diff --git a/.github/workflows/docs_check.yml b/.github/workflows/docs_check.yml index 2b02e7c23ae..0c657a245cb 100644 --- a/.github/workflows/docs_check.yml +++ b/.github/workflows/docs_check.yml @@ -13,9 +13,9 @@ on: # yamllint disable-line rule:truthy branches: - master paths: + - 'docker/docs/**' - 'docs/**' - 'website/**' - - 'docker/docs/**' jobs: CheckLabels: runs-on: [self-hosted, style-checker] diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6482ddebe06..01490dff59e 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -13,6 +13,7 @@ on: # yamllint disable-line rule:truthy branches: - master paths-ignore: + - 'docker/docs/**' - 'docs/**' - 'website/**' ########################################################################################## diff --git a/.gitmodules b/.gitmodules index 8b30973951f..55fd684fddb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -265,10 +265,6 @@ [submodule "contrib/wyhash"] path = contrib/wyhash url = https://github.com/wangyi-fudan/wyhash.git -[submodule "contrib/eigen"] - path = contrib/eigen - url = https://github.com/eigen-mirror/eigen [submodule "contrib/hashidsxx"] path = contrib/hashidsxx url = https://github.com/schoentoon/hashidsxx.git - diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e02ded4070..abe263834ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -351,6 +351,10 @@ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${C set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_C_FLAGS_ADD}") +set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} ${COMPILER_FLAGS} ${CMAKE_ASM_FLAGS_ADD}") +set (CMAKE_ASM_FLAGS_RELWITHDEBINFO "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}") +set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_ASM_FLAGS_ADD}") + if (COMPILER_CLANG) if (OS_DARWIN) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") diff --git a/SECURITY.md b/SECURITY.md index 6c03a6bb945..426559a0439 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -25,9 +25,11 @@ The following versions of ClickHouse server are currently being supported with s | 21.10 | :x: | | 21.11 | :x: | | 21.12 | :x: | -| 22.1 | ✅ | -| 22.2 | ✅ | +| 22.1 | :x: | +| 22.2 | :x: | | 22.3 | ✅ | +| 22.4 | ✅ | +| 22.5 | ✅ | ## Reporting a Vulnerability diff --git a/base/base/defines.h b/base/base/defines.h index bd98e99f5b9..084e710abf6 100644 --- a/base/base/defines.h +++ b/base/base/defines.h @@ -105,6 +105,25 @@ # define ASAN_POISON_MEMORY_REGION(a, b) #endif +#if !defined(ABORT_ON_LOGICAL_ERROR) + #if !defined(NDEBUG) || defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(UNDEFINED_BEHAVIOR_SANITIZER) + #define ABORT_ON_LOGICAL_ERROR + #endif +#endif + +/// chassert(x) is similar to assert(x), but: +/// - works in builds with sanitizers, not only in debug builds +/// - tries to print failed assertion into server log +/// It can be used for all assertions except heavy ones. +/// Heavy assertions (that run loops or call complex functions) are allowed in debug builds only. +#if !defined(chassert) + #if defined(ABORT_ON_LOGICAL_ERROR) + #define chassert(x) static_cast(x) ? void(0) : abortOnFailedAssertion(#x) + #else + #define chassert(x) ((void)0) + #endif +#endif + /// A template function for suppressing warnings about unused variables or function results. template constexpr void UNUSED(Args &&... args [[maybe_unused]]) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a7f1a908474..943e0e0ebc1 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -153,7 +153,6 @@ endif() add_contrib (sqlite-cmake sqlite-amalgamation) add_contrib (s2geometry-cmake s2geometry) -add_contrib (eigen-cmake eigen) # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear diff --git a/contrib/eigen b/contrib/eigen deleted file mode 160000 index 3147391d946..00000000000 --- a/contrib/eigen +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3147391d946bb4b6c68edd901f2add6ac1f31f8c diff --git a/contrib/eigen-cmake/CMakeLists.txt b/contrib/eigen-cmake/CMakeLists.txt deleted file mode 100644 index a37d341109c..00000000000 --- a/contrib/eigen-cmake/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -set(EIGEN_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/eigen") - -add_library (_eigen INTERFACE) - -# Only include MPL2 code from Eigen library -target_compile_definitions(_eigen INTERFACE EIGEN_MPL2_ONLY) - -# Clang by default mimics gcc 4.2.1 compatibility but Eigen checks __GNUC__ version to enable -# a workaround for bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867 fixed in 6.3 -# So we fake gcc > 6.3 when building with clang -if (COMPILER_CLANG AND ARCH_PPC64LE) - target_compile_options(_eigen INTERFACE -fgnuc-version=6.4) -endif() - -target_include_directories (_eigen SYSTEM INTERFACE ${EIGEN_LIBRARY_DIR}) -add_library(ch_contrib::eigen ALIAS _eigen) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 39ee7fd079d..c59b4da890b 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -170,6 +170,12 @@ endif () target_compile_definitions(_jemalloc PRIVATE -DJEMALLOC_PROF=1) if (USE_UNWIND) + # jemalloc provides support for two different libunwind flavors: the original HP libunwind and the one coming with gcc / g++ / libstdc++. + # The latter is identified by `JEMALLOC_PROF_LIBGCC` and uses `_Unwind_Backtrace` method instead of `unw_backtrace`. + # At the time ClickHouse uses LLVM libunwind which follows libgcc's way of backtracing. + + # ClickHouse has to provide `unw_backtrace` method by the means of [commit 8e2b31e](https://github.com/ClickHouse/libunwind/commit/8e2b31e766dd502f6df74909e04a7dbdf5182eb1). + target_compile_definitions (_jemalloc PRIVATE -DJEMALLOC_PROF_LIBGCC=1) target_link_libraries (_jemalloc PRIVATE unwind) endif () diff --git a/docker/docs/release/Dockerfile b/docker/docs/release/Dockerfile index 024cf8e6cc6..89536889746 100644 --- a/docker/docs/release/Dockerfile +++ b/docker/docs/release/Dockerfile @@ -1,4 +1,3 @@ -# rebuild in #33610 # docker build -t clickhouse/docs-release . FROM ubuntu:20.04 diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 3a660d9cf15..cafc62b365e 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -177,7 +177,6 @@ function clone_submodules contrib/jemalloc contrib/replxx contrib/wyhash - contrib/eigen contrib/hashidsxx ) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index d78de84f60d..94fdfd536a7 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -3,8 +3,6 @@ from multiprocessing import cpu_count from subprocess import Popen, call, check_output, STDOUT import os -import sys -import shutil import argparse import logging import time @@ -31,6 +29,9 @@ def get_options(i, backward_compatibility_check): if i % 5 == 1: client_options.append("join_use_nulls=1") + if i % 15 == 1: + client_options.append("join_algorithm='parallel_hash'") + if i % 15 == 6: client_options.append("join_algorithm='partial_merge'") diff --git a/docs/_includes/cmake_in_clickhouse_header.md b/docs/_includes/cmake_in_clickhouse_header.md index 02019f13964..c35668c2c40 100644 --- a/docs/_includes/cmake_in_clickhouse_header.md +++ b/docs/_includes/cmake_in_clickhouse_header.md @@ -9,11 +9,6 @@ cmake .. \ -DCMAKE_C_COMPILER=$(which clang-13) \ -DCMAKE_CXX_COMPILER=$(which clang++-13) \ -DCMAKE_BUILD_TYPE=Debug \ - -DENABLE_CLICKHOUSE_ALL=OFF \ - -DENABLE_CLICKHOUSE_SERVER=ON \ - -DENABLE_CLICKHOUSE_CLIENT=ON \ - -DENABLE_LIBRARIES=OFF \ - -DUSE_UNWIND=ON \ -DENABLE_UTILS=OFF \ -DENABLE_TESTS=OFF ``` diff --git a/docs/changelogs/v20.10.1.4881-prestable.md b/docs/changelogs/v20.10.1.4881-prestable.md new file mode 100644 index 00000000000..a40830ddb43 --- /dev/null +++ b/docs/changelogs/v20.10.1.4881-prestable.md @@ -0,0 +1,197 @@ +### ClickHouse release v20.10.1.4881-prestable FIXME as compared to v20.9.1.4585-prestable + +#### Backward Incompatible Change +* Add support for nested multiline comments `/* comment /* comment */ */` in SQL. This conforms to the SQL standard. [#14655](https://github.com/ClickHouse/ClickHouse/pull/14655) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change default value of `format_regexp_escaping_rule` setting (it's related to `Regexp` format) to `Raw` (it means - read whole subpattern as a value) to make the behaviour more like to what users expect. [#15426](https://github.com/ClickHouse/ClickHouse/pull/15426) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `multiple_joins_rewriter_version` obsolete. Remove first version of joins rewriter. [#15472](https://github.com/ClickHouse/ClickHouse/pull/15472) ([Artem Zuikov](https://github.com/4ertus2)). + +#### New Feature +* Supporting MySQL types: `decimal` (as ClickHouse `Decimal`) and `datetime` with sub-second precision (as `DateTime64`). ... [#11512](https://github.com/ClickHouse/ClickHouse/pull/11512) ([Vasily Nemkov](https://github.com/Enmk)). +* Allow to turn on fsync on inserts, merges and fetches. [#11948](https://github.com/ClickHouse/ClickHouse/pull/11948) ([Anton Popov](https://github.com/CurtizJ)). +* Secure inter-cluster query execution (with initial_user as current query user). [#13156](https://github.com/ClickHouse/ClickHouse/pull/13156) ([Azat Khuzhin](https://github.com/azat)). +* * Add `mapPopulateSeries` function. [#13166](https://github.com/ClickHouse/ClickHouse/pull/13166) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Allow user to specify settings for `ReplicatedMergeTree*` storage in `` section of config file. It works similarly to `` section. For `ReplicatedMergeTree*` storages settings from `` and `` are applied together, but settings from `` has higher priority. Added `system.replicated_merge_tree_settings` table. [#13573](https://github.com/ClickHouse/ClickHouse/pull/13573) ([Amos Bird](https://github.com/amosbird)). +* Add new feature: format LineAsString that accepts a sequence of line separated by newlines, spaces and/or commas. [#13846](https://github.com/ClickHouse/ClickHouse/pull/13846) ([hexiaoting](https://github.com/hexiaoting)). +* New query complexity limit settings `max_rows_to_read_leaf`, `max_bytes_to_read_leaf` for distributed queries to limit max rows/bytes read on the leaf nodes. Limit is applied for local reads only, **excluding** the final merge stage on the root node. [#14221](https://github.com/ClickHouse/ClickHouse/pull/14221) ([Roman Khavronenko](https://github.com/hagen1778)). +* Add JSONStrings formats which output data in arrays of strings. [#14333](https://github.com/ClickHouse/ClickHouse/pull/14333) ([hcz](https://github.com/hczhcz)). +* Now insert statements can have asterisk (or variants) with column transformers in the column list. [#14453](https://github.com/ClickHouse/ClickHouse/pull/14453) ([Amos Bird](https://github.com/amosbird)). +* Added a script to import git repository to ClickHouse. [#14471](https://github.com/ClickHouse/ClickHouse/pull/14471) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add the ability to specify `TTL ... RECOMPRESS codec_name` for MergeTree table engines family. [#14494](https://github.com/ClickHouse/ClickHouse/pull/14494) ([alesapin](https://github.com/alesapin)). +* Add event_time_microseconds to `system.asynchronous_metric_log` & `system.metric_log` tables. [#14514](https://github.com/ClickHouse/ClickHouse/pull/14514) ([Bharat Nallan](https://github.com/bharatnc)). +* Add new feature: SHOW DATABASES LIKE 'xxx'. [#14521](https://github.com/ClickHouse/ClickHouse/pull/14521) ([hexiaoting](https://github.com/hexiaoting)). +* Support decimal data type for MaterializedMySQL. [#14535](https://github.com/ClickHouse/ClickHouse/pull/14535) ([Winter Zhang](https://github.com/zhang2014)). +* Allow configurable NULL representation for TSV output format. It is controlled by the setting `output_format_tsv_null_representation` which is `\N` by default. This closes [#9375](https://github.com/ClickHouse/ClickHouse/issues/9375). Note that the setting only controls output format and `\N` is the only supported NULL representation for TSV input format. [#14586](https://github.com/ClickHouse/ClickHouse/pull/14586) ([Kruglov Pavel](https://github.com/Avogar)). +* Add new feature: format LineAsString that accepts a sequence of line separated by newlines, spaces and/or commas. [#14703](https://github.com/ClickHouse/ClickHouse/pull/14703) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added `formatReadableQuantity` function. It is useful for reading big numbers by human. [#14725](https://github.com/ClickHouse/ClickHouse/pull/14725) ([Artem Hnilov](https://github.com/BooBSD)). +* Add the ability to remove column properties and table TTLs. Introduced queries `ALTER TABLE MODIFY COLUMN col_name REMOVE what_to_remove` and `ALTER TABLE REMOVE TTL`. Both operations are lightweight and executed at the metadata level. [#14742](https://github.com/ClickHouse/ClickHouse/pull/14742) ([alesapin](https://github.com/alesapin)). +* Introduce event_time_microseconds field to `system.text_log`, `system.trace_log`, `system.query_log` and `system.query_thread_log` tables. [#14760](https://github.com/ClickHouse/ClickHouse/pull/14760) ([Bharat Nallan](https://github.com/bharatnc)). +* Now we support `WITH AS (subquery) ... ` to introduce named subqueries in the query context. This closes [#2416](https://github.com/ClickHouse/ClickHouse/issues/2416). This closes [#4967](https://github.com/ClickHouse/ClickHouse/issues/4967). [#14771](https://github.com/ClickHouse/ClickHouse/pull/14771) ([Amos Bird](https://github.com/amosbird)). +* Allow to omit arguments for Replicated table engine if defaults are specified in config. [#14791](https://github.com/ClickHouse/ClickHouse/pull/14791) ([vxider](https://github.com/Vxider)). +* Add table function `null('structure')`. [#14797](https://github.com/ClickHouse/ClickHouse/pull/14797) ([vxider](https://github.com/Vxider)). +* Added query obfuscation tool. It allows to share more queries for better testing. This closes [#15268](https://github.com/ClickHouse/ClickHouse/issues/15268). [#15321](https://github.com/ClickHouse/ClickHouse/pull/15321) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added format `RawBLOB`. It is intended for input or output a single value without any escaping and delimiters. This closes [#15349](https://github.com/ClickHouse/ClickHouse/issues/15349). [#15364](https://github.com/ClickHouse/ClickHouse/pull/15364) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix [15350](https://github.com/ClickHouse/ClickHouse/issues/15350). [#15443](https://github.com/ClickHouse/ClickHouse/pull/15443) ([flynn](https://github.com/ucasfl)). +* Introduce `enable_global_with_statement` setting which propagates the first select's `WITH` statements to other select queries at the same level, and makes aliases in `WITH` statements visible to subqueries. [#15451](https://github.com/ClickHouse/ClickHouse/pull/15451) ([Amos Bird](https://github.com/amosbird)). +* Add the `reinterpretAsUUID` function that allows to convert a big-endian byte string to UUID. [#15480](https://github.com/ClickHouse/ClickHouse/pull/15480) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add parallel quorum inserts. This closes [#15601](https://github.com/ClickHouse/ClickHouse/issues/15601). [#15601](https://github.com/ClickHouse/ClickHouse/pull/15601) ([Latysheva Alexandra](https://github.com/alexelex)). + +#### Performance Improvement +* Enable compact parts by default for small parts. This will allow to process frequent inserts slightly more efficiently (4..100 times). [#11913](https://github.com/ClickHouse/ClickHouse/pull/11913) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve performance of 256-bit bytes using (u)int64_t as base type for wide integers. Original wide integers use 8-bit types as base. [#14859](https://github.com/ClickHouse/ClickHouse/pull/14859) ([Artem Zuikov](https://github.com/4ertus2)). +* Only `mlock` code segment when starting clickhouse-server. In previous versions, all mapped regions were locked in memory, including debug info. Debug info is usually splitted to a separate file but if it isn't, it led to +2..3 GiB memory usage. [#14929](https://github.com/ClickHouse/ClickHouse/pull/14929) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* We used to choose fixed key method to group by one fixed string. It's unnecessary since we have StringHashTable which do the similar packedFix optimization for FixedString columns. And we should use low_cardinality_key_fixed_string if possible. [#15034](https://github.com/ClickHouse/ClickHouse/pull/15034) ([Amos Bird](https://github.com/amosbird)). +* Fix `DateTime DateTime` mistakenly choosing the slow generic implementation. This fixes [#15153](https://github.com/ClickHouse/ClickHouse/issues/15153) . [#15178](https://github.com/ClickHouse/ClickHouse/pull/15178) ([Amos Bird](https://github.com/amosbird)). +* Use one S3 DeleteObjects request instead of multiple DeleteObject in cycle. No any functionality changes, so covered by existing tests like integration/test_log_family_s3. [#15238](https://github.com/ClickHouse/ClickHouse/pull/15238) ([ianton-ru](https://github.com/ianton-ru)). +* Faster 256-bit multiplication. [#15418](https://github.com/ClickHouse/ClickHouse/pull/15418) ([Artem Zuikov](https://github.com/4ertus2)). +* Improve `quantileTDigest` performance. This fixes [#2668](https://github.com/ClickHouse/ClickHouse/issues/2668). [#15542](https://github.com/ClickHouse/ClickHouse/pull/15542) ([Kruglov Pavel](https://github.com/Avogar)). +* Explicitly use a temporary disk to store vertical merge temporary data. [#15639](https://github.com/ClickHouse/ClickHouse/pull/15639) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). + +#### Improvement +* When duplicate block is written to replica where it does not exist locally (has not been fetched from replicas), don't ignore it and write locally to achieve the same effect as if it was successfully replicated. [#11684](https://github.com/ClickHouse/ClickHouse/pull/11684) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support custom codecs in compact parts. [#12183](https://github.com/ClickHouse/ClickHouse/pull/12183) ([Anton Popov](https://github.com/CurtizJ)). +* Now joinGet supports multi-key lookup. Continuation of [#12418](https://github.com/ClickHouse/ClickHouse/issues/12418). [#13015](https://github.com/ClickHouse/ClickHouse/pull/13015) ([Amos Bird](https://github.com/amosbird)). +* For INSERTs with inline data in VALUES format, support semicolon as the data terminator, in addition to the new line. Closes [#12288](https://github.com/ClickHouse/ClickHouse/issues/12288). [#13192](https://github.com/ClickHouse/ClickHouse/pull/13192) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* SYSTEM RELOAD CONFIG now throws an exception if failed to reload and continues using the previous users.xml. The background periodic reloading also continues using the previous users.xml if failed to reload. [#14492](https://github.com/ClickHouse/ClickHouse/pull/14492) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add an option to skip access checks for DiskS3. [#14497](https://github.com/ClickHouse/ClickHouse/pull/14497) ([Pavel Kovalenko](https://github.com/Jokser)). +* ClickHouse treats partition expr and key expr differently. Partition expr is used to construct an minmax index containing related columns, while primary key expr is stored as an expr. Sometimes user might partition a table at coarser levels, such as `partition by i / 1000`. However, binary operators are not monotonic and this PR tries to fix that. It might also benifit other use cases. [#14513](https://github.com/ClickHouse/ClickHouse/pull/14513) ([Amos Bird](https://github.com/amosbird)). +* Fix some trailing whitespaces in query format. [#14595](https://github.com/ClickHouse/ClickHouse/pull/14595) ([Azat Khuzhin](https://github.com/azat)). +* Add `QueryMemoryLimitExceeded` event. This closes [#14589](https://github.com/ClickHouse/ClickHouse/issues/14589). [#14647](https://github.com/ClickHouse/ClickHouse/pull/14647) ([fastio](https://github.com/fastio)). +* Fixed the backward-incompatible change by providing the options to build without debug info for functions. [#14657](https://github.com/ClickHouse/ClickHouse/pull/14657) ([Mike Kot](https://github.com/myrrc)). +* dynamic reload zookeeper config. [#14678](https://github.com/ClickHouse/ClickHouse/pull/14678) ([sundyli](https://github.com/sundy-li)). +* Allow parallel execution of distributed DDL. [#14684](https://github.com/ClickHouse/ClickHouse/pull/14684) ([Azat Khuzhin](https://github.com/azat)). +* Fix potential memory leak caused by zookeeper exists watch. [#14693](https://github.com/ClickHouse/ClickHouse/pull/14693) ([hustnn](https://github.com/hustnn)). +* Fixed "Packet payload is not fully read" error in `MaterializeMySQL` database engine. [#14696](https://github.com/ClickHouse/ClickHouse/pull/14696) ([BohuTANG](https://github.com/BohuTANG)). +* Fix crash in `bitShiftLeft()` when called with negative big integer. [#14697](https://github.com/ClickHouse/ClickHouse/pull/14697) ([Artem Zuikov](https://github.com/4ertus2)). +* Add `merge_algorithm` to system.merges table to improve merging inspections. [#14705](https://github.com/ClickHouse/ClickHouse/pull/14705) ([Amos Bird](https://github.com/amosbird)). +* Less unneded code generated by DecimalBinaryOperation template in FunctionBinaryArithmetic. [#14743](https://github.com/ClickHouse/ClickHouse/pull/14743) ([Artem Zuikov](https://github.com/4ertus2)). +* Now columns can be used to wrap over a list of columns and apply column transformers afterwards. [#14775](https://github.com/ClickHouse/ClickHouse/pull/14775) ([Amos Bird](https://github.com/amosbird)). +* Support for disabling persistency for StorageJoin and StorageSet, this feature is controlled by setting `disable_set_and_join_persistency`. And this PR solved issue [#6318](https://github.com/ClickHouse/ClickHouse/issues/6318). [#14776](https://github.com/ClickHouse/ClickHouse/pull/14776) ([vxider](https://github.com/Vxider)). +* Construct `query_start_time` and `query_start_time_microseconds` from the same timespec. [#14831](https://github.com/ClickHouse/ClickHouse/pull/14831) ([Bharat Nallan](https://github.com/bharatnc)). +* Allow using multi-volume storage configuration in storage Distributed. [#14839](https://github.com/ClickHouse/ClickHouse/pull/14839) ([Pavel Kovalenko](https://github.com/Jokser)). +* Show subqueries for `SET` and `JOIN` in `EXPLAIN` result. [#14856](https://github.com/ClickHouse/ClickHouse/pull/14856) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Provide a `load_balancing_first_offset` query setting to explicitly state what the first replica is. It's used together with `FIRST_OR_RANDOM` load balancing strategy, which allows to control replicas workload. [#14867](https://github.com/ClickHouse/ClickHouse/pull/14867) ([Amos Bird](https://github.com/amosbird)). +* Fixed excessive settings constraint violation when running SELECT with SETTINGS from a distributed table. [#14876](https://github.com/ClickHouse/ClickHouse/pull/14876) ([Amos Bird](https://github.com/amosbird)). +* Allow to drop Replicated table if previous drop attempt was failed due to ZooKeeper session expiration. This fixes [#11891](https://github.com/ClickHouse/ClickHouse/issues/11891). [#14926](https://github.com/ClickHouse/ClickHouse/pull/14926) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid deadlock when executing INSERT SELECT into itself from a table with `TinyLog` or `Log` table engines. This closes [#6802](https://github.com/ClickHouse/ClickHouse/issues/6802). [#14962](https://github.com/ClickHouse/ClickHouse/pull/14962) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Ignore key constraints when doing mutations. Without this pr, it's not possible to do mutations when `force_index_by_date = 1` or `force_primary_key = 1`. [#14973](https://github.com/ClickHouse/ClickHouse/pull/14973) ([Amos Bird](https://github.com/amosbird)). +* Add option to disable TTL move on data part insert. [#15000](https://github.com/ClickHouse/ClickHouse/pull/15000) ([Pavel Kovalenko](https://github.com/Jokser)). +* Enable `Atomic` database engine by default. [#15003](https://github.com/ClickHouse/ClickHouse/pull/15003) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Proper exception message for wrong number of arguments of CAST. This closes [#13992](https://github.com/ClickHouse/ClickHouse/issues/13992). [#15029](https://github.com/ClickHouse/ClickHouse/pull/15029) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add the ability to specify specialized codecs like `Delta`, `T64`, etc. for columns with subtypes. Implements [#12551](https://github.com/ClickHouse/ClickHouse/issues/12551), fixes [#11397](https://github.com/ClickHouse/ClickHouse/issues/11397), fixes [#4609](https://github.com/ClickHouse/ClickHouse/issues/4609). [#15089](https://github.com/ClickHouse/ClickHouse/pull/15089) ([alesapin](https://github.com/alesapin)). +* Added `optimize` setting to `EXPLAIN PLAN` query. If enabled, query plan level optimisations are applied. Enabled by default. [#15201](https://github.com/ClickHouse/ClickHouse/pull/15201) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Do not allow connections to ClickHouse server until all scripts in `/docker-entrypoint-initdb.d/` are executed. [#15244](https://github.com/ClickHouse/ClickHouse/pull/15244) ([Aleksei Kozharin](https://github.com/alekseik1)). +* fix [15264](https://github.com/ClickHouse/ClickHouse/issues/15264). [#15285](https://github.com/ClickHouse/ClickHouse/pull/15285) ([flynn](https://github.com/ucasfl)). +* Unfold `{database}`, `{table}` and `{uuid}` macros in `zookeeper_path` on replicated table creation. Do not allow `RENAME TABLE` if it may break `zookeeper_path` after server restart. Fixes [#6917](https://github.com/ClickHouse/ClickHouse/issues/6917). [#15348](https://github.com/ClickHouse/ClickHouse/pull/15348) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add support for "Raw" column format for `Regexp` format. It allows to simply extract subpatterns as a whole without any escaping rules. [#15363](https://github.com/ClickHouse/ClickHouse/pull/15363) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now it's possible to change the type of version column for `VersionedCollapsingMergeTree` with `ALTER` query. [#15442](https://github.com/ClickHouse/ClickHouse/pull/15442) ([alesapin](https://github.com/alesapin)). +* Wait for `DROP/DETACH TABLE` to actually finish if `NO DELAY` or `SYNC` is specified for `Atomic` database. [#15448](https://github.com/ClickHouse/ClickHouse/pull/15448) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Pass through *_for_user settings via Distributed with cluster-secure. [#15551](https://github.com/ClickHouse/ClickHouse/pull/15551) ([Azat Khuzhin](https://github.com/azat)). +* Use experimental pass manager by default. [#15608](https://github.com/ClickHouse/ClickHouse/pull/15608) ([Daniel Kutenin](https://github.com/danlark1)). +* Implement force_data_skipping_indices setting. [#15642](https://github.com/ClickHouse/ClickHouse/pull/15642) ([Azat Khuzhin](https://github.com/azat)). + +#### Bug Fix +* Fix `currentDatabase()` function cannot be used in `ON CLUSTER` ddl query. [#14211](https://github.com/ClickHouse/ClickHouse/pull/14211) ([Winter Zhang](https://github.com/zhang2014)). +* Fixed the incorrect sorting order of `Nullable` column. This fixes [#14344](https://github.com/ClickHouse/ClickHouse/issues/14344). [#14495](https://github.com/ClickHouse/ClickHouse/pull/14495) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix executable dictionary source hang. In previous versions, when using some formats (e.g. `JSONEachRow`) data was not feed to a child process before it outputs at least something. This closes [#1697](https://github.com/ClickHouse/ClickHouse/issues/1697). This closes [#2455](https://github.com/ClickHouse/ClickHouse/issues/2455). [#14525](https://github.com/ClickHouse/ClickHouse/pull/14525) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a bug when converting Nullable String to Enum. Introduced by https://github.com/ClickHouse/ClickHouse/pull/12745 . This fixes [#14435](https://github.com/ClickHouse/ClickHouse/issues/14435) . [#14530](https://github.com/ClickHouse/ClickHouse/pull/14530) ([Amos Bird](https://github.com/amosbird)). +* Fix rare segfaults in functions with combinator -Resample, which could appear in result of overflow with very large parameters. [#14562](https://github.com/ClickHouse/ClickHouse/pull/14562) ([Anton Popov](https://github.com/CurtizJ)). +* Cleanup data directory after Zookeeper exceptions during CreateQuery for StorageReplicatedMergeTree Engine. [#14563](https://github.com/ClickHouse/ClickHouse/pull/14563) ([Bharat Nallan](https://github.com/bharatnc)). +* Added the checker as neither calling `lc->isNullable()` nor calling `ls->getDictionaryPtr()->isNullable()` would return the correct result. [#14591](https://github.com/ClickHouse/ClickHouse/pull/14591) ([Mike Kot](https://github.com/myrrc)). +* Fix wrong Decimal multiplication result caused wrong decimal scale of result column. [#14603](https://github.com/ClickHouse/ClickHouse/pull/14603) ([Artem Zuikov](https://github.com/4ertus2)). +* Stuff the query into ASTFunction's argument list so that we don't break the presumptions of some AST visitors. This fixes [#14608](https://github.com/ClickHouse/ClickHouse/issues/14608). [#14611](https://github.com/ClickHouse/ClickHouse/pull/14611) ([Amos Bird](https://github.com/amosbird)). +* Fix bug when `ALTER UPDATE` mutation with Nullable column in assignment expression and constant value (like `UPDATE x = 42`) leads to incorrect value in column or segfault. Fixes [#13634](https://github.com/ClickHouse/ClickHouse/issues/13634), [#14045](https://github.com/ClickHouse/ClickHouse/issues/14045). [#14646](https://github.com/ClickHouse/ClickHouse/pull/14646) ([alesapin](https://github.com/alesapin)). +* Fixed missed default database name in metadata of materialized view when executing `ALTER ... MODIFY QUERY`. [#14664](https://github.com/ClickHouse/ClickHouse/pull/14664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Replace column transformer should replace identifiers with cloned ASTs. This fixes [#14695](https://github.com/ClickHouse/ClickHouse/issues/14695) . [#14734](https://github.com/ClickHouse/ClickHouse/pull/14734) ([Amos Bird](https://github.com/amosbird)). +* Fix wrong monotonicity detection for shrunk `Int -> Int` cast of signed types. It might lead to incorrect query result. This bug is unveiled in [#14513](https://github.com/ClickHouse/ClickHouse/issues/14513). [#14783](https://github.com/ClickHouse/ClickHouse/pull/14783) ([Amos Bird](https://github.com/amosbird)). +* Fix unreleased bug for LineAsString Format. [#14842](https://github.com/ClickHouse/ClickHouse/pull/14842) ([hexiaoting](https://github.com/hexiaoting)). +* Fix a problem where the server may get stuck on startup while talking to ZooKeeper, if the configuration files have to be fetched from ZK (using the `from_zk` include option). This fixes [#14814](https://github.com/ClickHouse/ClickHouse/issues/14814). [#14843](https://github.com/ClickHouse/ClickHouse/pull/14843) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix rare error in `SELECT` queries when the queried column has `DEFAULT` expression which depends on the other column which also has `DEFAULT` and not present in select query and not exists on disk. Partially fixes [#14531](https://github.com/ClickHouse/ClickHouse/issues/14531). [#14845](https://github.com/ClickHouse/ClickHouse/pull/14845) ([alesapin](https://github.com/alesapin)). +* Fixed bug in parsing MySQL binlog events, which causes `Attempt to read after eof` and `Packet payload is not fully read` in `MaterializeMySQL` database engine. [#14852](https://github.com/ClickHouse/ClickHouse/pull/14852) ([Winter Zhang](https://github.com/zhang2014)). +* Fixed segfault in CacheDictionary [#14837](https://github.com/ClickHouse/ClickHouse/issues/14837). [#14879](https://github.com/ClickHouse/ClickHouse/pull/14879) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix SIGSEGV for an attempt to INSERT into StorageFile(fd). [#14887](https://github.com/ClickHouse/ClickHouse/pull/14887) ([Azat Khuzhin](https://github.com/azat)). +* Fix the issue when some invocations of `extractAllGroups` function may trigger "Memory limit exceeded" error. This fixes [#13383](https://github.com/ClickHouse/ClickHouse/issues/13383). [#14889](https://github.com/ClickHouse/ClickHouse/pull/14889) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed `.metadata.tmp File exists` error when using `MaterializeMySQL` database engine. [#14898](https://github.com/ClickHouse/ClickHouse/pull/14898) ([Winter Zhang](https://github.com/zhang2014)). +* Publish CPU frequencies per logical core in `system.asynchronous_metrics`. This fixes [#14923](https://github.com/ClickHouse/ClickHouse/issues/14923). [#14924](https://github.com/ClickHouse/ClickHouse/pull/14924) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix to make predicate push down work when subquery contains finalizeAggregation function. Fixes [#14847](https://github.com/ClickHouse/ClickHouse/issues/14847). [#14937](https://github.com/ClickHouse/ClickHouse/pull/14937) ([filimonov](https://github.com/filimonov)). +* Update jemalloc to fix possible issues with percpu arena. [#14957](https://github.com/ClickHouse/ClickHouse/pull/14957) ([Azat Khuzhin](https://github.com/azat)). +* Now settings `number_of_free_entries_in_pool_to_execute_mutation` and `number_of_free_entries_in_pool_to_lower_max_size_of_merge` can be equal to `background_pool_size`. [#14975](https://github.com/ClickHouse/ClickHouse/pull/14975) ([alesapin](https://github.com/alesapin)). +* Fix crash in RIGHT or FULL JOIN with join_algorith='auto' when memory limit exceeded and we should change HashJoin with MergeJoin. [#15002](https://github.com/ClickHouse/ClickHouse/pull/15002) ([Artem Zuikov](https://github.com/4ertus2)). +* Fixed `Cannot rename ... errno: 22, strerror: Invalid argument` error on DDL query execution in Atomic database when running clickhouse-server in docker on Mac OS. [#15024](https://github.com/ClickHouse/ClickHouse/pull/15024) ([Alexander Tokmakov](https://github.com/tavplubix)). +* If function `bar` was called with specifically crafted arguments, buffer overflow was possible. This closes [#13926](https://github.com/ClickHouse/ClickHouse/issues/13926). [#15028](https://github.com/ClickHouse/ClickHouse/pull/15028) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* We already use padded comparison between String and FixedString (https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionsComparison.h#L333). This PR applies the same logic to field comparison which corrects the usage of FixedString as primary keys. This fixes [#14908](https://github.com/ClickHouse/ClickHouse/issues/14908). [#15033](https://github.com/ClickHouse/ClickHouse/pull/15033) ([Amos Bird](https://github.com/amosbird)). +* Fixes `Data compressed with different methods` in `join_algorithm='auto'`. Keep LowCardinality as type for left table join key in `join_algorithm='partial_merge'`. [#15088](https://github.com/ClickHouse/ClickHouse/pull/15088) ([Artem Zuikov](https://github.com/4ertus2)). +* Adjust decimals field size in mysql column definition packet. [#15152](https://github.com/ClickHouse/ClickHouse/pull/15152) ([maqroll](https://github.com/maqroll)). +* Fix bug in table engine `Buffer` which doesn't allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). +* Fix instance crash when using joinGet with LowCardinality types. This fixes [#15214](https://github.com/ClickHouse/ClickHouse/issues/15214). [#15220](https://github.com/ClickHouse/ClickHouse/pull/15220) ([Amos Bird](https://github.com/amosbird)). +* Fix 'Unknown identifier' in GROUP BY when query has JOIN over Merge table. [#15242](https://github.com/ClickHouse/ClickHouse/pull/15242) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix MSan report in QueryLog. Uninitialized memory can be used for the field `memory_usage`. [#15258](https://github.com/ClickHouse/ClickHouse/pull/15258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix hang of queries with a lot of subqueries to same table of `MySQL` engine. Previously, if there were more than 16 subqueries to same `MySQL` table in query, it hang forever. [#15299](https://github.com/ClickHouse/ClickHouse/pull/15299) ([Anton Popov](https://github.com/CurtizJ)). +* Fix rare race condition on server startup when system.logs are enabled. [#15300](https://github.com/ClickHouse/ClickHouse/pull/15300) ([alesapin](https://github.com/alesapin)). +* Fix race condition during MergeTree table rename and background cleanup. [#15304](https://github.com/ClickHouse/ClickHouse/pull/15304) ([alesapin](https://github.com/alesapin)). +* Fix bug where queries like SELECT toStartOfDay(today()) fail complaining about empty time_zone argument. [#15319](https://github.com/ClickHouse/ClickHouse/pull/15319) ([Bharat Nallan](https://github.com/bharatnc)). +* Fixed compression in S3 storage. [#15376](https://github.com/ClickHouse/ClickHouse/pull/15376) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix multiple occurrences of column transformers in a select query. [#15378](https://github.com/ClickHouse/ClickHouse/pull/15378) ([Amos Bird](https://github.com/amosbird)). +* fixes [#15365](https://github.com/ClickHouse/ClickHouse/issues/15365) fix attach mysql database engine throw exception(no query context). [#15384](https://github.com/ClickHouse/ClickHouse/pull/15384) ([Winter Zhang](https://github.com/zhang2014)). +* Report proper error when the second argument of `boundingRatio` aggregate function has a wrong type. [#15407](https://github.com/ClickHouse/ClickHouse/pull/15407) ([detailyang](https://github.com/detailyang)). +* Fix bug with event subscription in DDLWorker which rarely may lead to query hangs in `ON CLUSTER`. Introduced in [#13450](https://github.com/ClickHouse/ClickHouse/issues/13450). [#15477](https://github.com/ClickHouse/ClickHouse/pull/15477) ([alesapin](https://github.com/alesapin)). +* Throw an error when a single parameter is passed to ReplicatedMergeTree instead of ignoring it. [#15516](https://github.com/ClickHouse/ClickHouse/pull/15516) ([nvartolomei](https://github.com/nvartolomei)). +* Fix `Missing columns` errors when selecting columns which absent in data, but depend on other columns which also absent in data. Fixes [#15530](https://github.com/ClickHouse/ClickHouse/issues/15530). [#15532](https://github.com/ClickHouse/ClickHouse/pull/15532) ([alesapin](https://github.com/alesapin)). +* Fix bug when `ILIKE` operator stops being case insensitive if `LIKE` with the same pattern was executed. [#15536](https://github.com/ClickHouse/ClickHouse/pull/15536) ([alesapin](https://github.com/alesapin)). +* Mutation might hang waiting for some non-existent part after `MOVE` or `REPLACE PARTITION` or, in rare cases, after `DETACH` or `DROP PARTITION`. It's fixed. [#15537](https://github.com/ClickHouse/ClickHouse/pull/15537) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 'Database doesn't exist.' in queries with IN and Distributed table when there's no database on initiator. [#15538](https://github.com/ClickHouse/ClickHouse/pull/15538) ([Artem Zuikov](https://github.com/4ertus2)). +* Significantly reduce memory usage in AggregatingInOrderTransform/optimize_aggregation_in_order. [#15543](https://github.com/ClickHouse/ClickHouse/pull/15543) ([Azat Khuzhin](https://github.com/azat)). +* Prevent the possibility of error message `Could not calculate available disk space (statvfs), errno: 4, strerror: Interrupted system call`. This fixes [#15541](https://github.com/ClickHouse/ClickHouse/issues/15541). [#15557](https://github.com/ClickHouse/ClickHouse/pull/15557) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Query is finished faster in case of exception. Cancel execution on remote replicas if exception happens. [#15578](https://github.com/ClickHouse/ClickHouse/pull/15578) ([Azat Khuzhin](https://github.com/azat)). +* Fixed `Element ... is not a constant expression` error when using `JSON*` function result in `VALUES`, `LIMIT` or right side of `IN` operator. [#15589](https://github.com/ClickHouse/ClickHouse/pull/15589) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix the order of destruction for resources in `ReadFromStorage` step of query plan. It might cause crashes in rare cases. Possibly connected with [#15610](https://github.com/ClickHouse/ClickHouse/issues/15610). [#15645](https://github.com/ClickHouse/ClickHouse/pull/15645) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Proper error handling during insert into MergeTree with S3. [#15657](https://github.com/ClickHouse/ClickHouse/pull/15657) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix race condition in AMQP-CPP. [#15667](https://github.com/ClickHouse/ClickHouse/pull/15667) ([alesapin](https://github.com/alesapin)). +* Fix rare race condition in dictionaries and tables from MySQL. [#15686](https://github.com/ClickHouse/ClickHouse/pull/15686) ([alesapin](https://github.com/alesapin)). +* Fixed too low default value of `max_replicated_logs_to_keep` setting, which might cause replicas to become lost too often. Improve lost replica recovery process by choosing the most up-to-date replica to clone. Also do not remove old parts from lost replica, detach them instead. [#15701](https://github.com/ClickHouse/ClickHouse/pull/15701) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error `Cannot find column` which may happen at insertion into `MATERIALIZED VIEW` in case if query for `MV` containes `ARRAY JOIN`. [#15717](https://github.com/ClickHouse/ClickHouse/pull/15717) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix some cases of queries, in which only virtual columns are selected. Previously `Not found column _nothing in block` exception may be thrown. Fixes [#12298](https://github.com/ClickHouse/ClickHouse/issues/12298). [#15756](https://github.com/ClickHouse/ClickHouse/pull/15756) ([Anton Popov](https://github.com/CurtizJ)). + +#### Build/Testing/Packaging Improvement +* Control CI builds configuration from the ClickHouse repository. [#14547](https://github.com/ClickHouse/ClickHouse/pull/14547) ([alesapin](https://github.com/alesapin)). +* Now ClickHouse uses gcc-10 for the release build. Fixes [#11138](https://github.com/ClickHouse/ClickHouse/issues/11138). [#14609](https://github.com/ClickHouse/ClickHouse/pull/14609) ([alesapin](https://github.com/alesapin)). +* Attempt to make performance test more reliable. It is done by remapping the executable memory of the process on the fly with `madvise` to use transparent huge pages - it can lower the number of iTLB misses which is the main source of instabilities in performance tests. [#14685](https://github.com/ClickHouse/ClickHouse/pull/14685) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* 1. In CMake files: - Moved some options' descriptions' parts to comments above. - Replace 0 -> `OFF`, 1 -> `ON` in `option`s default values. - Added some descriptions and links to docs to the options. - Replaced `FUZZER` option (there is another option `ENABLE_FUZZING` which also enables same functionality). - Removed `ENABLE_GTEST_LIBRARY` option as there is `ENABLE_TESTS`. [#14711](https://github.com/ClickHouse/ClickHouse/pull/14711) ([Mike Kot](https://github.com/myrrc)). +* Speed up build a little by removing unused headers. [#14714](https://github.com/ClickHouse/ClickHouse/pull/14714) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build failure in OSX. [#14761](https://github.com/ClickHouse/ClickHouse/pull/14761) ([Winter Zhang](https://github.com/zhang2014)). +* Attempt to speed up build a little. [#14808](https://github.com/ClickHouse/ClickHouse/pull/14808) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now we use clang-11 to build ClickHouse in CI. [#14846](https://github.com/ClickHouse/ClickHouse/pull/14846) ([alesapin](https://github.com/alesapin)). +* #14809 fix MaterializeMySQL empty transaction unstable test case found in CI. [#14854](https://github.com/ClickHouse/ClickHouse/pull/14854) ([Winter Zhang](https://github.com/zhang2014)). +* Reformat and cleanup code in all integration test *.py files. [#14864](https://github.com/ClickHouse/ClickHouse/pull/14864) ([Bharat Nallan](https://github.com/bharatnc)). +* Fixing tests/integration/test_distributed_over_live_view/test.py. [#14892](https://github.com/ClickHouse/ClickHouse/pull/14892) ([vzakaznikov](https://github.com/vzakaznikov)). +* Switch from `clang-tidy-10` to `clang-tidy-11`. [#14922](https://github.com/ClickHouse/ClickHouse/pull/14922) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Convert to python3. This closes [#14886](https://github.com/ClickHouse/ClickHouse/issues/14886). [#15007](https://github.com/ClickHouse/ClickHouse/pull/15007) ([Azat Khuzhin](https://github.com/azat)). +* Make performance test more stable and representative by splitting test runs and profile runs. [#15027](https://github.com/ClickHouse/ClickHouse/pull/15027) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Maybe fix MSan report in base64 (on servers with AVX-512). This fixes [#14006](https://github.com/ClickHouse/ClickHouse/issues/14006). [#15030](https://github.com/ClickHouse/ClickHouse/pull/15030) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't allow any C++ translation unit to build more than 10 minutes or to use more than 10 GB or memory. This fixes [#14925](https://github.com/ClickHouse/ClickHouse/issues/14925). [#15060](https://github.com/ClickHouse/ClickHouse/pull/15060) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now all test images use `llvm-symbolizer-11`. [#15069](https://github.com/ClickHouse/ClickHouse/pull/15069) ([alesapin](https://github.com/alesapin)). +* Splitted huge test `test_dictionaries_all_layouts_and_sources` into smaller ones. [#15110](https://github.com/ClickHouse/ClickHouse/pull/15110) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added a script to perform hardware benchmark in a single command. [#15115](https://github.com/ClickHouse/ClickHouse/pull/15115) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix CMake options forwarding in fast test script. Fixes error in [#14711](https://github.com/ClickHouse/ClickHouse/issues/14711). [#15155](https://github.com/ClickHouse/ClickHouse/pull/15155) ([alesapin](https://github.com/alesapin)). +* Improvements in CI docker images: get rid of ZooKeeper and single script for test configs installation. [#15215](https://github.com/ClickHouse/ClickHouse/pull/15215) ([alesapin](https://github.com/alesapin)). +* Now we use clang-11 for production ClickHouse build. [#15239](https://github.com/ClickHouse/ClickHouse/pull/15239) ([alesapin](https://github.com/alesapin)). +* Allow to run AArch64 version of clickhouse-server without configs. This facilitates [#15174](https://github.com/ClickHouse/ClickHouse/issues/15174). [#15266](https://github.com/ClickHouse/ClickHouse/pull/15266) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fail early in functional tests if server failed to respond. This closes [#15262](https://github.com/ClickHouse/ClickHouse/issues/15262). [#15267](https://github.com/ClickHouse/ClickHouse/pull/15267) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix bug for build error: [#15272](https://github.com/ClickHouse/ClickHouse/issues/15272). [#15297](https://github.com/ClickHouse/ClickHouse/pull/15297) ([hexiaoting](https://github.com/hexiaoting)). +* fix bug for building query_db_generator.cpp. [#15353](https://github.com/ClickHouse/ClickHouse/pull/15353) ([hexiaoting](https://github.com/hexiaoting)). +* Allow to build with llvm-11. [#15366](https://github.com/ClickHouse/ClickHouse/pull/15366) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Switch binary builds(Linux, Darwin, AArch64, FreeDSD) to clang-11. [#15622](https://github.com/ClickHouse/ClickHouse/pull/15622) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix build of one miscellaneous example tool on Mac OS. Note that we don't build examples on Mac OS in our CI (we build only ClickHouse binary), so there is zero chance it will not break again. This fixes [#15804](https://github.com/ClickHouse/ClickHouse/issues/15804). [#15808](https://github.com/ClickHouse/ClickHouse/pull/15808) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Other +* Make binary a bit smaller (~50 Mb for debug version). [#14555](https://github.com/ClickHouse/ClickHouse/pull/14555) ([Artem Zuikov](https://github.com/4ertus2)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Bump numpy from 1.19.1 to 1.19.2 in /docs/tools'. [#14733](https://github.com/ClickHouse/ClickHouse/pull/14733) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Modify the minimum version of the Clang compiler'. [#14882](https://github.com/ClickHouse/ClickHouse/pull/14882) ([Simon Liu](https://github.com/monadbobo)). +* NO CL ENTRY: 'fix a syntax error bug while using copier'. [#14890](https://github.com/ClickHouse/ClickHouse/pull/14890) ([HyaZz](https://github.com/HyaZz)). +* NO CL ENTRY: 'Bump mkdocs-macros-plugin from 0.4.9 to 0.4.13 in /docs/tools'. [#15067](https://github.com/ClickHouse/ClickHouse/pull/15067) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Revert "Test and doc for PR12771 krb5 + cyrus-sasl + kerberized kafka"'. [#15232](https://github.com/ClickHouse/ClickHouse/pull/15232) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Avoid deadlocks in Log/TinyLog"'. [#15259](https://github.com/ClickHouse/ClickHouse/pull/15259) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Bump mkdocs-macros-plugin from 0.4.13 to 0.4.17 in /docs/tools'. [#15460](https://github.com/ClickHouse/ClickHouse/pull/15460) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). + diff --git a/docs/changelogs/v20.10.2.20-stable.md b/docs/changelogs/v20.10.2.20-stable.md new file mode 100644 index 00000000000..3f03721fb16 --- /dev/null +++ b/docs/changelogs/v20.10.2.20-stable.md @@ -0,0 +1,180 @@ +### ClickHouse release v20.10.2.20-stable FIXME as compared to v20.9.1.4585-prestable + +#### Backward Incompatible Change +* Add support for nested multiline comments `/* comment /* comment */ */` in SQL. This conforms to the SQL standard. [#14655](https://github.com/ClickHouse/ClickHouse/pull/14655) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change default value of `format_regexp_escaping_rule` setting (it's related to `Regexp` format) to `Raw` (it means - read whole subpattern as a value) to make the behaviour more like to what users expect. [#15426](https://github.com/ClickHouse/ClickHouse/pull/15426) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `multiple_joins_rewriter_version` obsolete. Remove first version of joins rewriter. [#15472](https://github.com/ClickHouse/ClickHouse/pull/15472) ([Artem Zuikov](https://github.com/4ertus2)). + +#### New Feature +* Add new feature: format LineAsString that accepts a sequence of line separated by newlines, spaces and/or commas. [#14703](https://github.com/ClickHouse/ClickHouse/pull/14703) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added `formatReadableQuantity` function. It is useful for reading big numbers by human. [#14725](https://github.com/ClickHouse/ClickHouse/pull/14725) ([Artem Hnilov](https://github.com/BooBSD)). +* Add the ability to remove column properties and table TTLs. Introduced queries `ALTER TABLE MODIFY COLUMN col_name REMOVE what_to_remove` and `ALTER TABLE REMOVE TTL`. Both operations are lightweight and executed at the metadata level. [#14742](https://github.com/ClickHouse/ClickHouse/pull/14742) ([alesapin](https://github.com/alesapin)). +* Introduce event_time_microseconds field to `system.text_log`, `system.trace_log`, `system.query_log` and `system.query_thread_log` tables. [#14760](https://github.com/ClickHouse/ClickHouse/pull/14760) ([Bharat Nallan](https://github.com/bharatnc)). +* Now we support `WITH AS (subquery) ... ` to introduce named subqueries in the query context. This closes [#2416](https://github.com/ClickHouse/ClickHouse/issues/2416). This closes [#4967](https://github.com/ClickHouse/ClickHouse/issues/4967). [#14771](https://github.com/ClickHouse/ClickHouse/pull/14771) ([Amos Bird](https://github.com/amosbird)). +* Allow to omit arguments for Replicated table engine if defaults are specified in config. [#14791](https://github.com/ClickHouse/ClickHouse/pull/14791) ([vxider](https://github.com/Vxider)). +* Add table function `null('structure')`. [#14797](https://github.com/ClickHouse/ClickHouse/pull/14797) ([vxider](https://github.com/Vxider)). +* Added query obfuscation tool. It allows to share more queries for better testing. This closes [#15268](https://github.com/ClickHouse/ClickHouse/issues/15268). [#15321](https://github.com/ClickHouse/ClickHouse/pull/15321) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added format `RawBLOB`. It is intended for input or output a single value without any escaping and delimiters. This closes [#15349](https://github.com/ClickHouse/ClickHouse/issues/15349). [#15364](https://github.com/ClickHouse/ClickHouse/pull/15364) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix [15350](https://github.com/ClickHouse/ClickHouse/issues/15350). [#15443](https://github.com/ClickHouse/ClickHouse/pull/15443) ([flynn](https://github.com/ucasfl)). +* Introduce `enable_global_with_statement` setting which propagates the first select's `WITH` statements to other select queries at the same level, and makes aliases in `WITH` statements visible to subqueries. [#15451](https://github.com/ClickHouse/ClickHouse/pull/15451) ([Amos Bird](https://github.com/amosbird)). +* Add the `reinterpretAsUUID` function that allows to convert a big-endian byte string to UUID. [#15480](https://github.com/ClickHouse/ClickHouse/pull/15480) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add parallel quorum inserts. This closes [#15601](https://github.com/ClickHouse/ClickHouse/issues/15601). [#15601](https://github.com/ClickHouse/ClickHouse/pull/15601) ([Latysheva Alexandra](https://github.com/alexelex)). + +#### Performance Improvement +* Improve performance of 256-bit bytes using (u)int64_t as base type for wide integers. Original wide integers use 8-bit types as base. [#14859](https://github.com/ClickHouse/ClickHouse/pull/14859) ([Artem Zuikov](https://github.com/4ertus2)). +* Only `mlock` code segment when starting clickhouse-server. In previous versions, all mapped regions were locked in memory, including debug info. Debug info is usually splitted to a separate file but if it isn't, it led to +2..3 GiB memory usage. [#14929](https://github.com/ClickHouse/ClickHouse/pull/14929) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* We used to choose fixed key method to group by one fixed string. It's unnecessary since we have StringHashTable which do the similar packedFix optimization for FixedString columns. And we should use low_cardinality_key_fixed_string if possible. [#15034](https://github.com/ClickHouse/ClickHouse/pull/15034) ([Amos Bird](https://github.com/amosbird)). +* Fix `DateTime DateTime` mistakenly choosing the slow generic implementation. This fixes [#15153](https://github.com/ClickHouse/ClickHouse/issues/15153) . [#15178](https://github.com/ClickHouse/ClickHouse/pull/15178) ([Amos Bird](https://github.com/amosbird)). +* Use one S3 DeleteObjects request instead of multiple DeleteObject in cycle. No any functionality changes, so covered by existing tests like integration/test_log_family_s3. [#15238](https://github.com/ClickHouse/ClickHouse/pull/15238) ([ianton-ru](https://github.com/ianton-ru)). +* Faster 256-bit multiplication. [#15418](https://github.com/ClickHouse/ClickHouse/pull/15418) ([Artem Zuikov](https://github.com/4ertus2)). +* Improve `quantileTDigest` performance. This fixes [#2668](https://github.com/ClickHouse/ClickHouse/issues/2668). [#15542](https://github.com/ClickHouse/ClickHouse/pull/15542) ([Kruglov Pavel](https://github.com/Avogar)). +* Explicitly use a temporary disk to store vertical merge temporary data. [#15639](https://github.com/ClickHouse/ClickHouse/pull/15639) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). + +#### Improvement +* Add `QueryMemoryLimitExceeded` event. This closes [#14589](https://github.com/ClickHouse/ClickHouse/issues/14589). [#14647](https://github.com/ClickHouse/ClickHouse/pull/14647) ([fastio](https://github.com/fastio)). +* Fixed the backward-incompatible change by providing the options to build without debug info for functions. [#14657](https://github.com/ClickHouse/ClickHouse/pull/14657) ([Mike Kot](https://github.com/myrrc)). +* dynamic reload zookeeper config. [#14678](https://github.com/ClickHouse/ClickHouse/pull/14678) ([sundyli](https://github.com/sundy-li)). +* Allow parallel execution of distributed DDL. [#14684](https://github.com/ClickHouse/ClickHouse/pull/14684) ([Azat Khuzhin](https://github.com/azat)). +* Fix potential memory leak caused by zookeeper exists watch. [#14693](https://github.com/ClickHouse/ClickHouse/pull/14693) ([hustnn](https://github.com/hustnn)). +* Fixed "Packet payload is not fully read" error in `MaterializeMySQL` database engine. [#14696](https://github.com/ClickHouse/ClickHouse/pull/14696) ([BohuTANG](https://github.com/BohuTANG)). +* Fix crash in `bitShiftLeft()` when called with negative big integer. [#14697](https://github.com/ClickHouse/ClickHouse/pull/14697) ([Artem Zuikov](https://github.com/4ertus2)). +* Add `merge_algorithm` to system.merges table to improve merging inspections. [#14705](https://github.com/ClickHouse/ClickHouse/pull/14705) ([Amos Bird](https://github.com/amosbird)). +* Less unneded code generated by DecimalBinaryOperation template in FunctionBinaryArithmetic. [#14743](https://github.com/ClickHouse/ClickHouse/pull/14743) ([Artem Zuikov](https://github.com/4ertus2)). +* Now columns can be used to wrap over a list of columns and apply column transformers afterwards. [#14775](https://github.com/ClickHouse/ClickHouse/pull/14775) ([Amos Bird](https://github.com/amosbird)). +* Support for disabling persistency for StorageJoin and StorageSet, this feature is controlled by setting `disable_set_and_join_persistency`. And this PR solved issue [#6318](https://github.com/ClickHouse/ClickHouse/issues/6318). [#14776](https://github.com/ClickHouse/ClickHouse/pull/14776) ([vxider](https://github.com/Vxider)). +* Construct `query_start_time` and `query_start_time_microseconds` from the same timespec. [#14831](https://github.com/ClickHouse/ClickHouse/pull/14831) ([Bharat Nallan](https://github.com/bharatnc)). +* Allow using multi-volume storage configuration in storage Distributed. [#14839](https://github.com/ClickHouse/ClickHouse/pull/14839) ([Pavel Kovalenko](https://github.com/Jokser)). +* Show subqueries for `SET` and `JOIN` in `EXPLAIN` result. [#14856](https://github.com/ClickHouse/ClickHouse/pull/14856) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Provide a `load_balancing_first_offset` query setting to explicitly state what the first replica is. It's used together with `FIRST_OR_RANDOM` load balancing strategy, which allows to control replicas workload. [#14867](https://github.com/ClickHouse/ClickHouse/pull/14867) ([Amos Bird](https://github.com/amosbird)). +* Fixed excessive settings constraint violation when running SELECT with SETTINGS from a distributed table. [#14876](https://github.com/ClickHouse/ClickHouse/pull/14876) ([Amos Bird](https://github.com/amosbird)). +* Allow to drop Replicated table if previous drop attempt was failed due to ZooKeeper session expiration. This fixes [#11891](https://github.com/ClickHouse/ClickHouse/issues/11891). [#14926](https://github.com/ClickHouse/ClickHouse/pull/14926) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid deadlock when executing INSERT SELECT into itself from a table with `TinyLog` or `Log` table engines. This closes [#6802](https://github.com/ClickHouse/ClickHouse/issues/6802). [#14962](https://github.com/ClickHouse/ClickHouse/pull/14962) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Ignore key constraints when doing mutations. Without this pr, it's not possible to do mutations when `force_index_by_date = 1` or `force_primary_key = 1`. [#14973](https://github.com/ClickHouse/ClickHouse/pull/14973) ([Amos Bird](https://github.com/amosbird)). +* Add option to disable TTL move on data part insert. [#15000](https://github.com/ClickHouse/ClickHouse/pull/15000) ([Pavel Kovalenko](https://github.com/Jokser)). +* Enable `Atomic` database engine by default. [#15003](https://github.com/ClickHouse/ClickHouse/pull/15003) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Proper exception message for wrong number of arguments of CAST. This closes [#13992](https://github.com/ClickHouse/ClickHouse/issues/13992). [#15029](https://github.com/ClickHouse/ClickHouse/pull/15029) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add the ability to specify specialized codecs like `Delta`, `T64`, etc. for columns with subtypes. Implements [#12551](https://github.com/ClickHouse/ClickHouse/issues/12551), fixes [#11397](https://github.com/ClickHouse/ClickHouse/issues/11397), fixes [#4609](https://github.com/ClickHouse/ClickHouse/issues/4609). [#15089](https://github.com/ClickHouse/ClickHouse/pull/15089) ([alesapin](https://github.com/alesapin)). +* Added `optimize` setting to `EXPLAIN PLAN` query. If enabled, query plan level optimisations are applied. Enabled by default. [#15201](https://github.com/ClickHouse/ClickHouse/pull/15201) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Do not allow connections to ClickHouse server until all scripts in `/docker-entrypoint-initdb.d/` are executed. [#15244](https://github.com/ClickHouse/ClickHouse/pull/15244) ([Aleksei Kozharin](https://github.com/alekseik1)). +* fix [15264](https://github.com/ClickHouse/ClickHouse/issues/15264). [#15285](https://github.com/ClickHouse/ClickHouse/pull/15285) ([flynn](https://github.com/ucasfl)). +* Unfold `{database}`, `{table}` and `{uuid}` macros in `zookeeper_path` on replicated table creation. Do not allow `RENAME TABLE` if it may break `zookeeper_path` after server restart. Fixes [#6917](https://github.com/ClickHouse/ClickHouse/issues/6917). [#15348](https://github.com/ClickHouse/ClickHouse/pull/15348) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add support for "Raw" column format for `Regexp` format. It allows to simply extract subpatterns as a whole without any escaping rules. [#15363](https://github.com/ClickHouse/ClickHouse/pull/15363) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now it's possible to change the type of version column for `VersionedCollapsingMergeTree` with `ALTER` query. [#15442](https://github.com/ClickHouse/ClickHouse/pull/15442) ([alesapin](https://github.com/alesapin)). +* Wait for `DROP/DETACH TABLE` to actually finish if `NO DELAY` or `SYNC` is specified for `Atomic` database. [#15448](https://github.com/ClickHouse/ClickHouse/pull/15448) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Pass through *_for_user settings via Distributed with cluster-secure. [#15551](https://github.com/ClickHouse/ClickHouse/pull/15551) ([Azat Khuzhin](https://github.com/azat)). +* Use experimental pass manager by default. [#15608](https://github.com/ClickHouse/ClickHouse/pull/15608) ([Daniel Kutenin](https://github.com/danlark1)). +* Implement force_data_skipping_indices setting. [#15642](https://github.com/ClickHouse/ClickHouse/pull/15642) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16102](https://github.com/ClickHouse/ClickHouse/issues/16102): Now it's allowed to execute `ALTER ... ON CLUSTER` queries regardless of the `` setting in cluster config. [#16075](https://github.com/ClickHouse/ClickHouse/pull/16075) ([alesapin](https://github.com/alesapin)). + +#### Bug Fix +* Fix wrong Decimal multiplication result caused wrong decimal scale of result column. [#14603](https://github.com/ClickHouse/ClickHouse/pull/14603) ([Artem Zuikov](https://github.com/4ertus2)). +* Stuff the query into ASTFunction's argument list so that we don't break the presumptions of some AST visitors. This fixes [#14608](https://github.com/ClickHouse/ClickHouse/issues/14608). [#14611](https://github.com/ClickHouse/ClickHouse/pull/14611) ([Amos Bird](https://github.com/amosbird)). +* Fix bug when `ALTER UPDATE` mutation with Nullable column in assignment expression and constant value (like `UPDATE x = 42`) leads to incorrect value in column or segfault. Fixes [#13634](https://github.com/ClickHouse/ClickHouse/issues/13634), [#14045](https://github.com/ClickHouse/ClickHouse/issues/14045). [#14646](https://github.com/ClickHouse/ClickHouse/pull/14646) ([alesapin](https://github.com/alesapin)). +* Fixed missed default database name in metadata of materialized view when executing `ALTER ... MODIFY QUERY`. [#14664](https://github.com/ClickHouse/ClickHouse/pull/14664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Replace column transformer should replace identifiers with cloned ASTs. This fixes [#14695](https://github.com/ClickHouse/ClickHouse/issues/14695) . [#14734](https://github.com/ClickHouse/ClickHouse/pull/14734) ([Amos Bird](https://github.com/amosbird)). +* Fix wrong monotonicity detection for shrunk `Int -> Int` cast of signed types. It might lead to incorrect query result. This bug is unveiled in [#14513](https://github.com/ClickHouse/ClickHouse/issues/14513). [#14783](https://github.com/ClickHouse/ClickHouse/pull/14783) ([Amos Bird](https://github.com/amosbird)). +* Fix unreleased bug for LineAsString Format. [#14842](https://github.com/ClickHouse/ClickHouse/pull/14842) ([hexiaoting](https://github.com/hexiaoting)). +* Fix a problem where the server may get stuck on startup while talking to ZooKeeper, if the configuration files have to be fetched from ZK (using the `from_zk` include option). This fixes [#14814](https://github.com/ClickHouse/ClickHouse/issues/14814). [#14843](https://github.com/ClickHouse/ClickHouse/pull/14843) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix rare error in `SELECT` queries when the queried column has `DEFAULT` expression which depends on the other column which also has `DEFAULT` and not present in select query and not exists on disk. Partially fixes [#14531](https://github.com/ClickHouse/ClickHouse/issues/14531). [#14845](https://github.com/ClickHouse/ClickHouse/pull/14845) ([alesapin](https://github.com/alesapin)). +* Fixed bug in parsing MySQL binlog events, which causes `Attempt to read after eof` and `Packet payload is not fully read` in `MaterializeMySQL` database engine. [#14852](https://github.com/ClickHouse/ClickHouse/pull/14852) ([Winter Zhang](https://github.com/zhang2014)). +* Fixed segfault in CacheDictionary [#14837](https://github.com/ClickHouse/ClickHouse/issues/14837). [#14879](https://github.com/ClickHouse/ClickHouse/pull/14879) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix SIGSEGV for an attempt to INSERT into StorageFile(fd). [#14887](https://github.com/ClickHouse/ClickHouse/pull/14887) ([Azat Khuzhin](https://github.com/azat)). +* Fix the issue when some invocations of `extractAllGroups` function may trigger "Memory limit exceeded" error. This fixes [#13383](https://github.com/ClickHouse/ClickHouse/issues/13383). [#14889](https://github.com/ClickHouse/ClickHouse/pull/14889) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed `.metadata.tmp File exists` error when using `MaterializeMySQL` database engine. [#14898](https://github.com/ClickHouse/ClickHouse/pull/14898) ([Winter Zhang](https://github.com/zhang2014)). +* Publish CPU frequencies per logical core in `system.asynchronous_metrics`. This fixes [#14923](https://github.com/ClickHouse/ClickHouse/issues/14923). [#14924](https://github.com/ClickHouse/ClickHouse/pull/14924) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix to make predicate push down work when subquery contains finalizeAggregation function. Fixes [#14847](https://github.com/ClickHouse/ClickHouse/issues/14847). [#14937](https://github.com/ClickHouse/ClickHouse/pull/14937) ([filimonov](https://github.com/filimonov)). +* Update jemalloc to fix possible issues with percpu arena. [#14957](https://github.com/ClickHouse/ClickHouse/pull/14957) ([Azat Khuzhin](https://github.com/azat)). +* Now settings `number_of_free_entries_in_pool_to_execute_mutation` and `number_of_free_entries_in_pool_to_lower_max_size_of_merge` can be equal to `background_pool_size`. [#14975](https://github.com/ClickHouse/ClickHouse/pull/14975) ([alesapin](https://github.com/alesapin)). +* Fix crash in RIGHT or FULL JOIN with join_algorith='auto' when memory limit exceeded and we should change HashJoin with MergeJoin. [#15002](https://github.com/ClickHouse/ClickHouse/pull/15002) ([Artem Zuikov](https://github.com/4ertus2)). +* Fixed `Cannot rename ... errno: 22, strerror: Invalid argument` error on DDL query execution in Atomic database when running clickhouse-server in docker on Mac OS. [#15024](https://github.com/ClickHouse/ClickHouse/pull/15024) ([Alexander Tokmakov](https://github.com/tavplubix)). +* If function `bar` was called with specifically crafted arguments, buffer overflow was possible. This closes [#13926](https://github.com/ClickHouse/ClickHouse/issues/13926). [#15028](https://github.com/ClickHouse/ClickHouse/pull/15028) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* We already use padded comparison between String and FixedString (https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionsComparison.h#L333). This PR applies the same logic to field comparison which corrects the usage of FixedString as primary keys. This fixes [#14908](https://github.com/ClickHouse/ClickHouse/issues/14908). [#15033](https://github.com/ClickHouse/ClickHouse/pull/15033) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15829](https://github.com/ClickHouse/ClickHouse/issues/15829): Update jemalloc to fix percpu_arena with affinity mask. [#15035](https://github.com/ClickHouse/ClickHouse/pull/15035) ([Azat Khuzhin](https://github.com/azat)). +* Fixes `Data compressed with different methods` in `join_algorithm='auto'`. Keep LowCardinality as type for left table join key in `join_algorithm='partial_merge'`. [#15088](https://github.com/ClickHouse/ClickHouse/pull/15088) ([Artem Zuikov](https://github.com/4ertus2)). +* Adjust decimals field size in mysql column definition packet. [#15152](https://github.com/ClickHouse/ClickHouse/pull/15152) ([maqroll](https://github.com/maqroll)). +* Fix bug in table engine `Buffer` which doesn't allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). +* Fix instance crash when using joinGet with LowCardinality types. This fixes [#15214](https://github.com/ClickHouse/ClickHouse/issues/15214). [#15220](https://github.com/ClickHouse/ClickHouse/pull/15220) ([Amos Bird](https://github.com/amosbird)). +* Fix 'Unknown identifier' in GROUP BY when query has JOIN over Merge table. [#15242](https://github.com/ClickHouse/ClickHouse/pull/15242) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix MSan report in QueryLog. Uninitialized memory can be used for the field `memory_usage`. [#15258](https://github.com/ClickHouse/ClickHouse/pull/15258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix hang of queries with a lot of subqueries to same table of `MySQL` engine. Previously, if there were more than 16 subqueries to same `MySQL` table in query, it hang forever. [#15299](https://github.com/ClickHouse/ClickHouse/pull/15299) ([Anton Popov](https://github.com/CurtizJ)). +* Fix rare race condition on server startup when system.logs are enabled. [#15300](https://github.com/ClickHouse/ClickHouse/pull/15300) ([alesapin](https://github.com/alesapin)). +* Fix race condition during MergeTree table rename and background cleanup. [#15304](https://github.com/ClickHouse/ClickHouse/pull/15304) ([alesapin](https://github.com/alesapin)). +* Fix bug where queries like SELECT toStartOfDay(today()) fail complaining about empty time_zone argument. [#15319](https://github.com/ClickHouse/ClickHouse/pull/15319) ([Bharat Nallan](https://github.com/bharatnc)). +* Fixed compression in S3 storage. [#15376](https://github.com/ClickHouse/ClickHouse/pull/15376) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix multiple occurrences of column transformers in a select query. [#15378](https://github.com/ClickHouse/ClickHouse/pull/15378) ([Amos Bird](https://github.com/amosbird)). +* fixes [#15365](https://github.com/ClickHouse/ClickHouse/issues/15365) fix attach mysql database engine throw exception(no query context). [#15384](https://github.com/ClickHouse/ClickHouse/pull/15384) ([Winter Zhang](https://github.com/zhang2014)). +* Report proper error when the second argument of `boundingRatio` aggregate function has a wrong type. [#15407](https://github.com/ClickHouse/ClickHouse/pull/15407) ([detailyang](https://github.com/detailyang)). +* Fix bug with event subscription in DDLWorker which rarely may lead to query hangs in `ON CLUSTER`. Introduced in [#13450](https://github.com/ClickHouse/ClickHouse/issues/13450). [#15477](https://github.com/ClickHouse/ClickHouse/pull/15477) ([alesapin](https://github.com/alesapin)). +* Throw an error when a single parameter is passed to ReplicatedMergeTree instead of ignoring it. [#15516](https://github.com/ClickHouse/ClickHouse/pull/15516) ([nvartolomei](https://github.com/nvartolomei)). +* Fix `Missing columns` errors when selecting columns which absent in data, but depend on other columns which also absent in data. Fixes [#15530](https://github.com/ClickHouse/ClickHouse/issues/15530). [#15532](https://github.com/ClickHouse/ClickHouse/pull/15532) ([alesapin](https://github.com/alesapin)). +* Fix bug when `ILIKE` operator stops being case insensitive if `LIKE` with the same pattern was executed. [#15536](https://github.com/ClickHouse/ClickHouse/pull/15536) ([alesapin](https://github.com/alesapin)). +* Mutation might hang waiting for some non-existent part after `MOVE` or `REPLACE PARTITION` or, in rare cases, after `DETACH` or `DROP PARTITION`. It's fixed. [#15537](https://github.com/ClickHouse/ClickHouse/pull/15537) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 'Database doesn't exist.' in queries with IN and Distributed table when there's no database on initiator. [#15538](https://github.com/ClickHouse/ClickHouse/pull/15538) ([Artem Zuikov](https://github.com/4ertus2)). +* Significantly reduce memory usage in AggregatingInOrderTransform/optimize_aggregation_in_order. [#15543](https://github.com/ClickHouse/ClickHouse/pull/15543) ([Azat Khuzhin](https://github.com/azat)). +* Prevent the possibility of error message `Could not calculate available disk space (statvfs), errno: 4, strerror: Interrupted system call`. This fixes [#15541](https://github.com/ClickHouse/ClickHouse/issues/15541). [#15557](https://github.com/ClickHouse/ClickHouse/pull/15557) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Query is finished faster in case of exception. Cancel execution on remote replicas if exception happens. [#15578](https://github.com/ClickHouse/ClickHouse/pull/15578) ([Azat Khuzhin](https://github.com/azat)). +* Fixed `Element ... is not a constant expression` error when using `JSON*` function result in `VALUES`, `LIMIT` or right side of `IN` operator. [#15589](https://github.com/ClickHouse/ClickHouse/pull/15589) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix the order of destruction for resources in `ReadFromStorage` step of query plan. It might cause crashes in rare cases. Possibly connected with [#15610](https://github.com/ClickHouse/ClickHouse/issues/15610). [#15645](https://github.com/ClickHouse/ClickHouse/pull/15645) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#16009](https://github.com/ClickHouse/ClickHouse/issues/16009): Fixed bug with globs in S3 table function, region from URL was not applied to S3 client configuration. [#15646](https://github.com/ClickHouse/ClickHouse/pull/15646) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Proper error handling during insert into MergeTree with S3. [#15657](https://github.com/ClickHouse/ClickHouse/pull/15657) ([Pavel Kovalenko](https://github.com/Jokser)). +* Backported in [#15869](https://github.com/ClickHouse/ClickHouse/issues/15869): Fix error `Cannot add simple transform to empty Pipe` which happened while reading from `Buffer` table which has different structure than destination table. It was possible if destination table returned empty result for query. Fixes [#15529](https://github.com/ClickHouse/ClickHouse/issues/15529). [#15662](https://github.com/ClickHouse/ClickHouse/pull/15662) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race condition in AMQP-CPP. [#15667](https://github.com/ClickHouse/ClickHouse/pull/15667) ([alesapin](https://github.com/alesapin)). +* Fix rare race condition in dictionaries and tables from MySQL. [#15686](https://github.com/ClickHouse/ClickHouse/pull/15686) ([alesapin](https://github.com/alesapin)). +* Fixed too low default value of `max_replicated_logs_to_keep` setting, which might cause replicas to become lost too often. Improve lost replica recovery process by choosing the most up-to-date replica to clone. Also do not remove old parts from lost replica, detach them instead. [#15701](https://github.com/ClickHouse/ClickHouse/pull/15701) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error `Cannot find column` which may happen at insertion into `MATERIALIZED VIEW` in case if query for `MV` containes `ARRAY JOIN`. [#15717](https://github.com/ClickHouse/ClickHouse/pull/15717) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#15926](https://github.com/ClickHouse/ClickHouse/issues/15926): Fix drop of materialized view with inner table in Atomic database (hangs all subsequent DROP TABLE due to hang of the worker thread, due to recursive DROP TABLE for inner table of MV). [#15743](https://github.com/ClickHouse/ClickHouse/pull/15743) ([Azat Khuzhin](https://github.com/azat)). +* Fix some cases of queries, in which only virtual columns are selected. Previously `Not found column _nothing in block` exception may be thrown. Fixes [#12298](https://github.com/ClickHouse/ClickHouse/issues/12298). [#15756](https://github.com/ClickHouse/ClickHouse/pull/15756) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#15867](https://github.com/ClickHouse/ClickHouse/issues/15867): Fix `select count()` inaccuracy for MaterializeMySQL. [#15767](https://github.com/ClickHouse/ClickHouse/pull/15767) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15902](https://github.com/ClickHouse/ClickHouse/issues/15902): Fix exception `Block structure mismatch` in `SELECT ... ORDER BY DESC` queries which were executed after `ALTER MODIFY COLUMN` query. Fixes [#15800](https://github.com/ClickHouse/ClickHouse/issues/15800). [#15852](https://github.com/ClickHouse/ClickHouse/pull/15852) ([alesapin](https://github.com/alesapin)). +* Backported in [#15922](https://github.com/ClickHouse/ClickHouse/issues/15922): Now exception will be thrown when `ALTER MODIFY COLUMN ... DEFAULT ...` has incompatible default with column type. Fixes [#15854](https://github.com/ClickHouse/ClickHouse/issues/15854). [#15858](https://github.com/ClickHouse/ClickHouse/pull/15858) ([alesapin](https://github.com/alesapin)). +* Backported in [#15917](https://github.com/ClickHouse/ClickHouse/issues/15917): Fix possible deadlocks in RBAC. [#15875](https://github.com/ClickHouse/ClickHouse/pull/15875) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#15950](https://github.com/ClickHouse/ClickHouse/issues/15950): fixes [#12513](https://github.com/ClickHouse/ClickHouse/issues/12513) fix difference expressions with same alias when analyze queries again. [#15886](https://github.com/ClickHouse/ClickHouse/pull/15886) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#16169](https://github.com/ClickHouse/ClickHouse/issues/16169): Fix incorrect empty result for query from `Distributed` table if query has `WHERE`, `PREWHERE` and `GLOBAL IN`. Fixes [#15792](https://github.com/ClickHouse/ClickHouse/issues/15792). [#15933](https://github.com/ClickHouse/ClickHouse/pull/15933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#15969](https://github.com/ClickHouse/ClickHouse/issues/15969): Fix a crash when database creation fails. [#15954](https://github.com/ClickHouse/ClickHouse/pull/15954) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#16023](https://github.com/ClickHouse/ClickHouse/issues/16023): Fix ambiguity in parsing of settings profiles: `CREATE USER ... SETTINGS profile readonly` is now considered as using a profile named `readonly`, not a setting named `profile` with the readonly constraint. This fixes [#15628](https://github.com/ClickHouse/ClickHouse/issues/15628). [#15982](https://github.com/ClickHouse/ClickHouse/pull/15982) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#16217](https://github.com/ClickHouse/ClickHouse/issues/16217): Fix rare segfaults when inserting into or selecting from MaterializedView and concurrently dropping target table (for Atomic database engine). [#15984](https://github.com/ClickHouse/ClickHouse/pull/15984) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#16027](https://github.com/ClickHouse/ClickHouse/issues/16027): Prevent replica hang for 5-10 mins when replication error happens after a period of inactivity. [#15987](https://github.com/ClickHouse/ClickHouse/pull/15987) ([filimonov](https://github.com/filimonov)). +* Backported in [#16089](https://github.com/ClickHouse/ClickHouse/issues/16089): Allow to use direct layout for dictionaries with complex keys. [#16007](https://github.com/ClickHouse/ClickHouse/pull/16007) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#16142](https://github.com/ClickHouse/ClickHouse/issues/16142): Fix `ALTER MODIFY ... ORDER BY` query hang for `ReplicatedVersionedCollapsingMergeTree`. This fixes [#15980](https://github.com/ClickHouse/ClickHouse/issues/15980). [#16011](https://github.com/ClickHouse/ClickHouse/pull/16011) ([alesapin](https://github.com/alesapin)). +* Backported in [#16077](https://github.com/ClickHouse/ClickHouse/issues/16077): Fixes [#15780](https://github.com/ClickHouse/ClickHouse/issues/15780) regression, e.g. indexOf([1, 2, 3], toLowCardinality(1)) now is prohibited but it should not be. [#16038](https://github.com/ClickHouse/ClickHouse/pull/16038) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#16120](https://github.com/ClickHouse/ClickHouse/issues/16120): Fix segfault in some cases of wrong aggregation in lambdas. [#16082](https://github.com/ClickHouse/ClickHouse/pull/16082) ([Anton Popov](https://github.com/CurtizJ)). + +#### Build/Testing/Packaging Improvement +* Now ClickHouse uses gcc-10 for the release build. Fixes [#11138](https://github.com/ClickHouse/ClickHouse/issues/11138). [#14609](https://github.com/ClickHouse/ClickHouse/pull/14609) ([alesapin](https://github.com/alesapin)). +* Attempt to make performance test more reliable. It is done by remapping the executable memory of the process on the fly with `madvise` to use transparent huge pages - it can lower the number of iTLB misses which is the main source of instabilities in performance tests. [#14685](https://github.com/ClickHouse/ClickHouse/pull/14685) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* 1. In CMake files: - Moved some options' descriptions' parts to comments above. - Replace 0 -> `OFF`, 1 -> `ON` in `option`s default values. - Added some descriptions and links to docs to the options. - Replaced `FUZZER` option (there is another option `ENABLE_FUZZING` which also enables same functionality). - Removed `ENABLE_GTEST_LIBRARY` option as there is `ENABLE_TESTS`. [#14711](https://github.com/ClickHouse/ClickHouse/pull/14711) ([Mike Kot](https://github.com/myrrc)). +* Speed up build a little by removing unused headers. [#14714](https://github.com/ClickHouse/ClickHouse/pull/14714) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build failure in OSX. [#14761](https://github.com/ClickHouse/ClickHouse/pull/14761) ([Winter Zhang](https://github.com/zhang2014)). +* Attempt to speed up build a little. [#14808](https://github.com/ClickHouse/ClickHouse/pull/14808) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now we use clang-11 to build ClickHouse in CI. [#14846](https://github.com/ClickHouse/ClickHouse/pull/14846) ([alesapin](https://github.com/alesapin)). +* #14809 fix MaterializeMySQL empty transaction unstable test case found in CI. [#14854](https://github.com/ClickHouse/ClickHouse/pull/14854) ([Winter Zhang](https://github.com/zhang2014)). +* Reformat and cleanup code in all integration test *.py files. [#14864](https://github.com/ClickHouse/ClickHouse/pull/14864) ([Bharat Nallan](https://github.com/bharatnc)). +* Fixing tests/integration/test_distributed_over_live_view/test.py. [#14892](https://github.com/ClickHouse/ClickHouse/pull/14892) ([vzakaznikov](https://github.com/vzakaznikov)). +* Switch from `clang-tidy-10` to `clang-tidy-11`. [#14922](https://github.com/ClickHouse/ClickHouse/pull/14922) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Convert to python3. This closes [#14886](https://github.com/ClickHouse/ClickHouse/issues/14886). [#15007](https://github.com/ClickHouse/ClickHouse/pull/15007) ([Azat Khuzhin](https://github.com/azat)). +* Make performance test more stable and representative by splitting test runs and profile runs. [#15027](https://github.com/ClickHouse/ClickHouse/pull/15027) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Maybe fix MSan report in base64 (on servers with AVX-512). This fixes [#14006](https://github.com/ClickHouse/ClickHouse/issues/14006). [#15030](https://github.com/ClickHouse/ClickHouse/pull/15030) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't allow any C++ translation unit to build more than 10 minutes or to use more than 10 GB or memory. This fixes [#14925](https://github.com/ClickHouse/ClickHouse/issues/14925). [#15060](https://github.com/ClickHouse/ClickHouse/pull/15060) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now all test images use `llvm-symbolizer-11`. [#15069](https://github.com/ClickHouse/ClickHouse/pull/15069) ([alesapin](https://github.com/alesapin)). +* Splitted huge test `test_dictionaries_all_layouts_and_sources` into smaller ones. [#15110](https://github.com/ClickHouse/ClickHouse/pull/15110) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added a script to perform hardware benchmark in a single command. [#15115](https://github.com/ClickHouse/ClickHouse/pull/15115) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix CMake options forwarding in fast test script. Fixes error in [#14711](https://github.com/ClickHouse/ClickHouse/issues/14711). [#15155](https://github.com/ClickHouse/ClickHouse/pull/15155) ([alesapin](https://github.com/alesapin)). +* Improvements in CI docker images: get rid of ZooKeeper and single script for test configs installation. [#15215](https://github.com/ClickHouse/ClickHouse/pull/15215) ([alesapin](https://github.com/alesapin)). +* Now we use clang-11 for production ClickHouse build. [#15239](https://github.com/ClickHouse/ClickHouse/pull/15239) ([alesapin](https://github.com/alesapin)). +* Allow to run AArch64 version of clickhouse-server without configs. This facilitates [#15174](https://github.com/ClickHouse/ClickHouse/issues/15174). [#15266](https://github.com/ClickHouse/ClickHouse/pull/15266) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fail early in functional tests if server failed to respond. This closes [#15262](https://github.com/ClickHouse/ClickHouse/issues/15262). [#15267](https://github.com/ClickHouse/ClickHouse/pull/15267) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix bug for build error: [#15272](https://github.com/ClickHouse/ClickHouse/issues/15272). [#15297](https://github.com/ClickHouse/ClickHouse/pull/15297) ([hexiaoting](https://github.com/hexiaoting)). +* fix bug for building query_db_generator.cpp. [#15353](https://github.com/ClickHouse/ClickHouse/pull/15353) ([hexiaoting](https://github.com/hexiaoting)). +* Allow to build with llvm-11. [#15366](https://github.com/ClickHouse/ClickHouse/pull/15366) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Switch binary builds(Linux, Darwin, AArch64, FreeDSD) to clang-11. [#15622](https://github.com/ClickHouse/ClickHouse/pull/15622) ([Ilya Yatsishin](https://github.com/qoega)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Bump numpy from 1.19.1 to 1.19.2 in /docs/tools'. [#14733](https://github.com/ClickHouse/ClickHouse/pull/14733) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Modify the minimum version of the Clang compiler'. [#14882](https://github.com/ClickHouse/ClickHouse/pull/14882) ([Simon Liu](https://github.com/monadbobo)). +* NO CL ENTRY: 'fix a syntax error bug while using copier'. [#14890](https://github.com/ClickHouse/ClickHouse/pull/14890) ([HyaZz](https://github.com/HyaZz)). +* NO CL ENTRY: 'Bump mkdocs-macros-plugin from 0.4.9 to 0.4.13 in /docs/tools'. [#15067](https://github.com/ClickHouse/ClickHouse/pull/15067) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Revert "Test and doc for PR12771 krb5 + cyrus-sasl + kerberized kafka"'. [#15232](https://github.com/ClickHouse/ClickHouse/pull/15232) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Avoid deadlocks in Log/TinyLog"'. [#15259](https://github.com/ClickHouse/ClickHouse/pull/15259) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Bump mkdocs-macros-plugin from 0.4.13 to 0.4.17 in /docs/tools'. [#15460](https://github.com/ClickHouse/ClickHouse/pull/15460) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). + diff --git a/docs/changelogs/v20.10.3.30-stable.md b/docs/changelogs/v20.10.3.30-stable.md new file mode 100644 index 00000000000..29fa85a4077 --- /dev/null +++ b/docs/changelogs/v20.10.3.30-stable.md @@ -0,0 +1,17 @@ +### ClickHouse release v20.10.3.30-stable FIXME as compared to v20.10.2.20-stable + +#### Improvement +* Backported in [#16313](https://github.com/ClickHouse/ClickHouse/issues/16313): Add allow_nondeterministic_optimize_skip_unused_shards (to allow non deterministic like rand() or dictGet() in sharding key). [#16105](https://github.com/ClickHouse/ClickHouse/pull/16105) ([Azat Khuzhin](https://github.com/azat)). + +#### Bug Fix +* Backported in [#16202](https://github.com/ClickHouse/ClickHouse/issues/16202): Decrement the `ReadonlyReplica` metric when detaching read-only tables. This fixes [#15598](https://github.com/ClickHouse/ClickHouse/issues/15598). [#15592](https://github.com/ClickHouse/ClickHouse/pull/15592) ([sundyli](https://github.com/sundy-li)). +* Backported in [#16177](https://github.com/ClickHouse/ClickHouse/issues/16177): Possibility to move part to another disk/volume if the first attempt was failed. [#15723](https://github.com/ClickHouse/ClickHouse/pull/15723) ([Pavel Kovalenko](https://github.com/Jokser)). +* Backported in [#16323](https://github.com/ClickHouse/ClickHouse/issues/16323): Fixed `DROP TABLE IF EXISTS` failure with `Table ... doesn't exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) Fixed `DROP/DETACH DATABASE` failure with `Table ... doesn't exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#16359](https://github.com/ClickHouse/ClickHouse/issues/16359): Fix collate name & charset name parser and support `length = 0` for string type. [#16008](https://github.com/ClickHouse/ClickHouse/pull/16008) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#16299](https://github.com/ClickHouse/ClickHouse/issues/16299): Fix dictGet in sharding_key (and similar places, i.e. when the function context is stored permanently). [#16205](https://github.com/ClickHouse/ClickHouse/pull/16205) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16234](https://github.com/ClickHouse/ClickHouse/issues/16234): Fix the case when memory can be overallocated regardless to the limit. This closes [#14560](https://github.com/ClickHouse/ClickHouse/issues/14560). [#16206](https://github.com/ClickHouse/ClickHouse/pull/16206) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#16327](https://github.com/ClickHouse/ClickHouse/issues/16327): Fix a possible memory leak during `GROUP BY` with string keys, caused by an error in `TwoLevelStringHashTable` implementation. [#16264](https://github.com/ClickHouse/ClickHouse/pull/16264) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#16374](https://github.com/ClickHouse/ClickHouse/issues/16374): Fix async Distributed INSERT w/ prefer_localhost_replica=0 and internal_replication. [#16358](https://github.com/ClickHouse/ClickHouse/pull/16358) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16419](https://github.com/ClickHouse/ClickHouse/issues/16419): Fix group by with totals/rollup/cube modifers and min/max functions over group by keys. Fixes [#16393](https://github.com/ClickHouse/ClickHouse/issues/16393). [#16397](https://github.com/ClickHouse/ClickHouse/pull/16397) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#16448](https://github.com/ClickHouse/ClickHouse/issues/16448): Fix double free in case of exception in function `dictGet`. It could have happened if dictionary was loaded with error. [#16429](https://github.com/ClickHouse/ClickHouse/pull/16429) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v20.10.4.1-stable.md b/docs/changelogs/v20.10.4.1-stable.md new file mode 100644 index 00000000000..cfebce93df6 --- /dev/null +++ b/docs/changelogs/v20.10.4.1-stable.md @@ -0,0 +1,17 @@ +### ClickHouse release v20.10.4.1-stable FIXME as compared to v20.10.3.30-stable + +#### Improvement +* Workaround for use S3 with nginx server as proxy. Nginx currenty does not accept urls with empty path like http://domain.com?delete, but vanilla aws-sdk-cpp produces this kind of urls. This commit uses patched aws-sdk-cpp version, which makes urls with "/" as path in this cases, like http://domain.com/?delete. [#16813](https://github.com/ClickHouse/ClickHouse/pull/16813) ([ianton-ru](https://github.com/ianton-ru)). + +#### Bug Fix +* Backported in [#16492](https://github.com/ClickHouse/ClickHouse/issues/16492): Fix bug with MySQL database. When MySQL server used as database engine is down some queries raise Exception, because they try to get tables from disabled server, while it's unnecessary. For example, query `SELECT ... FROM system.parts` should work only with MergeTree tables and don't touch MySQL database at all. [#16032](https://github.com/ClickHouse/ClickHouse/pull/16032) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#16710](https://github.com/ClickHouse/ClickHouse/issues/16710): Fixed the inconsistent behaviour when a part of return data could be dropped because the set for its filtration wasn't created. [#16308](https://github.com/ClickHouse/ClickHouse/pull/16308) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#16505](https://github.com/ClickHouse/ClickHouse/issues/16505): Fix processing of very large entries in replication queue. Very large entries may appear in ALTER queries if table structure is extremely large (near 1 MB). This fixes [#16307](https://github.com/ClickHouse/ClickHouse/issues/16307). [#16332](https://github.com/ClickHouse/ClickHouse/pull/16332) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#16471](https://github.com/ClickHouse/ClickHouse/issues/16471): Fix DROP TABLE for Distributed (racy with INSERT). [#16409](https://github.com/ClickHouse/ClickHouse/pull/16409) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16572](https://github.com/ClickHouse/ClickHouse/issues/16572): Fix rapid growth of metadata when using MySQL Master -> MySQL Slave -> ClickHouse MaterializeMySQL Engine, and `slave_parallel_worker` enabled on MySQL Slave, by properly shrinking GTID sets. This fixes [#15951](https://github.com/ClickHouse/ClickHouse/issues/15951). [#16504](https://github.com/ClickHouse/ClickHouse/pull/16504) ([TCeason](https://github.com/TCeason)). +* Backported in [#16551](https://github.com/ClickHouse/ClickHouse/issues/16551): Now when parsing AVRO from input the LowCardinality is removed from type. Fixes [#16188](https://github.com/ClickHouse/ClickHouse/issues/16188). [#16521](https://github.com/ClickHouse/ClickHouse/pull/16521) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#16749](https://github.com/ClickHouse/ClickHouse/issues/16749): Fixed [#16081](https://github.com/ClickHouse/ClickHouse/issues/16081). [#16613](https://github.com/ClickHouse/ClickHouse/pull/16613) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#16760](https://github.com/ClickHouse/ClickHouse/issues/16760): This will fix optimize_read_in_order/optimize_aggregation_in_order with max_threads>0 and expression in ORDER BY. [#16637](https://github.com/ClickHouse/ClickHouse/pull/16637) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16741](https://github.com/ClickHouse/ClickHouse/issues/16741): Fix `IN` operator over several columns and tuples with enabled `transform_null_in` setting. Fixes [#15310](https://github.com/ClickHouse/ClickHouse/issues/15310). [#16722](https://github.com/ClickHouse/ClickHouse/pull/16722) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#16893](https://github.com/ClickHouse/ClickHouse/issues/16893): Fix rare silent crashes when query profiler is on and ClickHouse is installed on OS with glibc version that has (supposedly) broken asynchronous unwind tables for some functions. This fixes [#15301](https://github.com/ClickHouse/ClickHouse/issues/15301). This fixes [#13098](https://github.com/ClickHouse/ClickHouse/issues/13098). [#16846](https://github.com/ClickHouse/ClickHouse/pull/16846) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.10.5.10-stable.md b/docs/changelogs/v20.10.5.10-stable.md new file mode 100644 index 00000000000..bdbabe8a03b --- /dev/null +++ b/docs/changelogs/v20.10.5.10-stable.md @@ -0,0 +1,18 @@ +### ClickHouse release v20.10.5.10-stable FIXME as compared to v20.10.4.1-stable + +#### Improvement +* Backported in [#17031](https://github.com/ClickHouse/ClickHouse/issues/17031): Make it possible to connect to `clickhouse-server` secure endpoint which requires SNI. This is possible when `clickhouse-server` is hosted behind TLS proxy. [#16938](https://github.com/ClickHouse/ClickHouse/pull/16938) ([filimonov](https://github.com/filimonov)). + +#### Bug Fix +* Backported in [#17107](https://github.com/ClickHouse/ClickHouse/issues/17107): fixes [#16574](https://github.com/ClickHouse/ClickHouse/issues/16574) fixes [#16231](https://github.com/ClickHouse/ClickHouse/issues/16231) fix remote query failure when using 'if' suffix aggregate function. [#16610](https://github.com/ClickHouse/ClickHouse/pull/16610) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#17022](https://github.com/ClickHouse/ClickHouse/issues/17022): Fix crash when using `any` without any arguments. This is for [#16803](https://github.com/ClickHouse/ClickHouse/issues/16803) . cc @azat. [#16826](https://github.com/ClickHouse/ClickHouse/pull/16826) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#16878](https://github.com/ClickHouse/ClickHouse/issues/16878): Abort multipart upload if no data was written to WriteBufferFromS3. [#16840](https://github.com/ClickHouse/ClickHouse/pull/16840) ([Pavel Kovalenko](https://github.com/Jokser)). +* Backported in [#16949](https://github.com/ClickHouse/ClickHouse/issues/16949): Prevent clickhouse server crashes when using TimeSeriesGroupSum. [#16865](https://github.com/ClickHouse/ClickHouse/pull/16865) ([filimonov](https://github.com/filimonov)). +* Backported in [#17075](https://github.com/ClickHouse/ClickHouse/issues/17075): Fix possible error `Illegal type of argument` for queries with `ORDER BY`. Fixes [#16580](https://github.com/ClickHouse/ClickHouse/issues/16580). [#16928](https://github.com/ClickHouse/ClickHouse/pull/16928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17008](https://github.com/ClickHouse/ClickHouse/issues/17008): Install script should always create subdirs in config folders. This is only relevant for Docker build with custom config. [#16936](https://github.com/ClickHouse/ClickHouse/pull/16936) ([filimonov](https://github.com/filimonov)). +* Backported in [#16967](https://github.com/ClickHouse/ClickHouse/issues/16967): Blame info was not calculated correctly in `clickhouse-git-import`. [#16959](https://github.com/ClickHouse/ClickHouse/pull/16959) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17012](https://github.com/ClickHouse/ClickHouse/issues/17012): Fix possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17038](https://github.com/ClickHouse/ClickHouse/issues/17038): Reresolve the IP of the `format_avro_schema_registry_url` in case of errors. [#16985](https://github.com/ClickHouse/ClickHouse/pull/16985) ([filimonov](https://github.com/filimonov)). +* Backported in [#17092](https://github.com/ClickHouse/ClickHouse/issues/17092): Fixed wrong result in big integers (128, 256 bit) when casting from double. [#16986](https://github.com/ClickHouse/ClickHouse/pull/16986) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#17169](https://github.com/ClickHouse/ClickHouse/issues/17169): Fix bug when `ON CLUSTER` queries may hang forever for non-leader ReplicatedMergeTreeTables. [#17089](https://github.com/ClickHouse/ClickHouse/pull/17089) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v20.10.6.27-stable.md b/docs/changelogs/v20.10.6.27-stable.md new file mode 100644 index 00000000000..68b34411927 --- /dev/null +++ b/docs/changelogs/v20.10.6.27-stable.md @@ -0,0 +1,31 @@ +### ClickHouse release v20.10.6.27-stable FIXME as compared to v20.10.5.10-stable + +#### Performance Improvement +* Backported in [#17591](https://github.com/ClickHouse/ClickHouse/issues/17591): Fix performance of reading from `Merge` tables over huge number of `MergeTree` tables. Fixes [#7748](https://github.com/ClickHouse/ClickHouse/issues/7748). [#16988](https://github.com/ClickHouse/ClickHouse/pull/16988) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix +* Backported in [#17157](https://github.com/ClickHouse/ClickHouse/issues/17157): Fixed uncontrolled growth of TDigest. [#16680](https://github.com/ClickHouse/ClickHouse/pull/16680) ([hrissan](https://github.com/hrissan)). +* Backported in [#17314](https://github.com/ClickHouse/ClickHouse/issues/17314): Return number of affected rows for INSERT queries via MySQL protocol. Previously ClickHouse used to always return 0, it's fixed. Fixes [#16605](https://github.com/ClickHouse/ClickHouse/issues/16605). [#16715](https://github.com/ClickHouse/ClickHouse/pull/16715) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#17341](https://github.com/ClickHouse/ClickHouse/issues/17341): TODO. [#16866](https://github.com/ClickHouse/ClickHouse/pull/16866) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17587](https://github.com/ClickHouse/ClickHouse/issues/17587): Fix optimization of group by with enabled setting `optimize_aggregators_of_group_by_keys` and joins. Fixes [#12604](https://github.com/ClickHouse/ClickHouse/issues/12604). [#16951](https://github.com/ClickHouse/ClickHouse/pull/16951) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17594](https://github.com/ClickHouse/ClickHouse/issues/17594): Fix order by optimization with monotonous functions. Fixes [#16107](https://github.com/ClickHouse/ClickHouse/issues/16107). [#16956](https://github.com/ClickHouse/ClickHouse/pull/16956) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17197](https://github.com/ClickHouse/ClickHouse/issues/17197): Avoid unnecessary network errors for remote queries which may be cancelled while execution, like queries with `LIMIT`. [#17006](https://github.com/ClickHouse/ClickHouse/pull/17006) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#17431](https://github.com/ClickHouse/ClickHouse/issues/17431): Bug fix for funciton fuzzBits, related issue: [#16980](https://github.com/ClickHouse/ClickHouse/issues/16980). [#17051](https://github.com/ClickHouse/ClickHouse/pull/17051) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#17130](https://github.com/ClickHouse/ClickHouse/issues/17130): Fixed crash on `CREATE TABLE ... AS some_table` query when `some_table` was created `AS table_function()` Fixes [#16944](https://github.com/ClickHouse/ClickHouse/issues/16944). [#17072](https://github.com/ClickHouse/ClickHouse/pull/17072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17193](https://github.com/ClickHouse/ClickHouse/issues/17193): Fix ColumnConst comparison which leads to crash. This fixed [#17088](https://github.com/ClickHouse/ClickHouse/issues/17088) . [#17135](https://github.com/ClickHouse/ClickHouse/pull/17135) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17395](https://github.com/ClickHouse/ClickHouse/issues/17395): Fix [#15235](https://github.com/ClickHouse/ClickHouse/issues/15235). When clickhouse-copier handle non-partitioned table, throws segfault error. [#17248](https://github.com/ClickHouse/ClickHouse/pull/17248) ([Qi Chen](https://github.com/kaka11chen)). +* Backported in [#17407](https://github.com/ClickHouse/ClickHouse/issues/17407): Fix set index invalidation when there are const columns in the subquery. This fixes [#17246](https://github.com/ClickHouse/ClickHouse/issues/17246) . [#17249](https://github.com/ClickHouse/ClickHouse/pull/17249) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17487](https://github.com/ClickHouse/ClickHouse/issues/17487): Fix crash while reading from `JOIN` table with `LowCardinality` types. Fixes [#17228](https://github.com/ClickHouse/ClickHouse/issues/17228). [#17397](https://github.com/ClickHouse/ClickHouse/pull/17397) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17491](https://github.com/ClickHouse/ClickHouse/issues/17491): Fix duplicates after `DISTINCT` which were possible because of incorrect optimization. Fixes [#17294](https://github.com/ClickHouse/ClickHouse/issues/17294). [#17296](https://github.com/ClickHouse/ClickHouse/pull/17296) ([li chengxiang](https://github.com/chengxianglibra)). [#17439](https://github.com/ClickHouse/ClickHouse/pull/17439) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17524](https://github.com/ClickHouse/ClickHouse/issues/17524): Fix `ORDER BY` with enabled setting `optimize_redundant_functions_in_order_by`. [#17471](https://github.com/ClickHouse/ClickHouse/pull/17471) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17532](https://github.com/ClickHouse/ClickHouse/issues/17532): Fix bug when mark cache size was underestimated by clickhouse. It may happen when there are a lot of tiny files with marks. [#17496](https://github.com/ClickHouse/ClickHouse/pull/17496) ([alesapin](https://github.com/alesapin)). +* Backported in [#17626](https://github.com/ClickHouse/ClickHouse/issues/17626): Fix alter query hang when the corresponding mutation was killed on the different replica. Fixes [#16953](https://github.com/ClickHouse/ClickHouse/issues/16953). [#17499](https://github.com/ClickHouse/ClickHouse/pull/17499) ([alesapin](https://github.com/alesapin)). +* Backported in [#17609](https://github.com/ClickHouse/ClickHouse/issues/17609): When clickhouse-client is used in interactive mode with multiline queries, single line comment was erronously extended till the end of query. This fixes [#13654](https://github.com/ClickHouse/ClickHouse/issues/13654). [#17565](https://github.com/ClickHouse/ClickHouse/pull/17565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17698](https://github.com/ClickHouse/ClickHouse/issues/17698): In might be determined incorrectly if cluster is circular- (cross-) replicated or not when executing `ON CLUSTER` query due to race condition when `pool_size` > 1. It's fixed. [#17640](https://github.com/ClickHouse/ClickHouse/pull/17640) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17730](https://github.com/ClickHouse/ClickHouse/issues/17730): Fixed `Function not implemented` error when executing `RENAME` query in `Atomic` database with ClickHouse running on Windows Subsystem for Linux. Fixes [#17661](https://github.com/ClickHouse/ClickHouse/issues/17661). [#17664](https://github.com/ClickHouse/ClickHouse/pull/17664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17783](https://github.com/ClickHouse/ClickHouse/issues/17783): Fixed problem when ClickHouse fails to resume connection to MySQL servers. [#17681](https://github.com/ClickHouse/ClickHouse/pull/17681) ([Alexander Kazakov](https://github.com/Akazz)). +* Backported in [#17816](https://github.com/ClickHouse/ClickHouse/issues/17816): Do not restore parts from WAL if `in_memory_parts_enable_wal` is disabled. [#17802](https://github.com/ClickHouse/ClickHouse/pull/17802) ([detailyang](https://github.com/detailyang)). + +#### Build/Testing/Packaging Improvement +* Backported in [#17289](https://github.com/ClickHouse/ClickHouse/issues/17289): Update embedded timezone data to version 2020d (also update cctz to the latest master). [#17204](https://github.com/ClickHouse/ClickHouse/pull/17204) ([filimonov](https://github.com/filimonov)). + diff --git a/docs/changelogs/v20.10.7.4-stable.md b/docs/changelogs/v20.10.7.4-stable.md new file mode 100644 index 00000000000..ae9dd8e53fb --- /dev/null +++ b/docs/changelogs/v20.10.7.4-stable.md @@ -0,0 +1,13 @@ +### ClickHouse release v20.10.7.4-stable FIXME as compared to v20.10.6.27-stable + +#### Bug Fix +* Backported in [#17798](https://github.com/ClickHouse/ClickHouse/issues/17798): - Fix optimize_distributed_group_by_sharding_key for query with OFFSET only. [#16996](https://github.com/ClickHouse/ClickHouse/pull/16996) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#18395](https://github.com/ClickHouse/ClickHouse/issues/18395): Fix empty `system.stack_trace` table when server is running in daemon mode. [#17630](https://github.com/ClickHouse/ClickHouse/pull/17630) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#18042](https://github.com/ClickHouse/ClickHouse/issues/18042): Fix possible segfault in `topK` aggregate function. This closes [#17404](https://github.com/ClickHouse/ClickHouse/issues/17404). [#17845](https://github.com/ClickHouse/ClickHouse/pull/17845) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#17980](https://github.com/ClickHouse/ClickHouse/issues/17980): fixes [#15187](https://github.com/ClickHouse/ClickHouse/issues/15187) fixes [#17912](https://github.com/ClickHouse/ClickHouse/issues/17912) support convert MySQL prefix index for MaterializeMySQL CC: @tavplubix. [#17944](https://github.com/ClickHouse/ClickHouse/pull/17944) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#18079](https://github.com/ClickHouse/ClickHouse/issues/18079): Fixed `std::out_of_range: basic_string` in S3 URL parsing. [#18059](https://github.com/ClickHouse/ClickHouse/pull/18059) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#18178](https://github.com/ClickHouse/ClickHouse/issues/18178): Fix `Unknown setting profile` error on attempt to set settings profile. [#18167](https://github.com/ClickHouse/ClickHouse/pull/18167) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#18361](https://github.com/ClickHouse/ClickHouse/issues/18361): fixes [#18186](https://github.com/ClickHouse/ClickHouse/issues/18186) fixes [#16372](https://github.com/ClickHouse/ClickHouse/issues/16372) fix unique key convert crash in MaterializeMySQL database engine. [#18211](https://github.com/ClickHouse/ClickHouse/pull/18211) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#18292](https://github.com/ClickHouse/ClickHouse/issues/18292): Fix key comparison between Enum and Int types. This fixes [#17989](https://github.com/ClickHouse/ClickHouse/issues/17989). [#18214](https://github.com/ClickHouse/ClickHouse/pull/18214) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#18295](https://github.com/ClickHouse/ClickHouse/issues/18295): - Fixed issue when `clickhouse-odbc-bridge` process is unreachable by server on machines with dual IPv4/IPv6 stack; - Fixed issue when ODBC dictionary updates are performed using malformed queries and/or cause crashes; Possibly closes [#14489](https://github.com/ClickHouse/ClickHouse/issues/14489). [#18278](https://github.com/ClickHouse/ClickHouse/pull/18278) ([Denis Glazachev](https://github.com/traceon)). + diff --git a/docs/changelogs/v20.11.1.5109-prestable.md b/docs/changelogs/v20.11.1.5109-prestable.md new file mode 100644 index 00000000000..0ebe0bf8b55 --- /dev/null +++ b/docs/changelogs/v20.11.1.5109-prestable.md @@ -0,0 +1,157 @@ +### ClickHouse release v20.11.1.5109-prestable FIXME as compared to v20.10.1.4881-prestable + +#### Backward Incompatible Change +* Make rankCorr function return nan on insufficient data [#16124](https://github.com/ClickHouse/ClickHouse/issues/16124). [#16135](https://github.com/ClickHouse/ClickHouse/pull/16135) ([hexiaoting](https://github.com/hexiaoting)). +* Aggregate functions `boundingRatio`, `rankCorr`, `retention`, `timeSeriesGroupSum`, `timeSeriesGroupRateSum`, `windowFunnel` were erroneously made case-insensitive. Now their names are made case sensitive as designed. Only functions that are specified in SQL standard or made for compatibility with other DBMS or functions similar to those should be case-insensitive. [#16407](https://github.com/ClickHouse/ClickHouse/pull/16407) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove `ANALYZE` and `AST` queries, and make the setting `enable_debug_queries` obsolete since now it is the part of full featured `EXPLAIN` query. [#16536](https://github.com/ClickHouse/ClickHouse/pull/16536) ([Ivan](https://github.com/abyss7)). +* Restrict to use of non-comparable data types (like `AggregateFunction`) in keys (Sorting key, Primary key, Partition key, and so on). [#16601](https://github.com/ClickHouse/ClickHouse/pull/16601) ([alesapin](https://github.com/alesapin)). +* If some `profile` was specified in `distributed_ddl` config section, then this profile could overwrite settings of `default` profile on server startup. It's fixed, now settings of distributed DDL queries should not affect global server settings. [#16635](https://github.com/ClickHouse/ClickHouse/pull/16635) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### New Feature +* #WelchTTest aggregate function implementation. [#10351](https://github.com/ClickHouse/ClickHouse/pull/10351) ([antikvist](https://github.com/antikvist)). +* New functions `encrypt`, `aes_encrypt_mysql`, `decrypt`, `aes_decrypt_mysql`. These functions are working slowly (below ClickHouse standards), so we consider it as an experimental feature. [#11844](https://github.com/ClickHouse/ClickHouse/pull/11844) ([Vasily Nemkov](https://github.com/Enmk)). +* - Added support of LDAP as a user directory for locally non-existent users. [#12736](https://github.com/ClickHouse/ClickHouse/pull/12736) ([Denis Glazachev](https://github.com/traceon)). +* Added `disable_merges` option for volumes in multi-disk configuration. [#13956](https://github.com/ClickHouse/ClickHouse/pull/13956) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Added initial OpenTelemetry support. ClickHouse now accepts OpenTelemetry traceparent headers over Native and HTTP protocols, and passes them downstream in some cases. The trace spans for executed queries are saved into the `system.opentelemetry_span_log` table. [#14195](https://github.com/ClickHouse/ClickHouse/pull/14195) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allows to read/write Single protobuf message at once (w/o length-delimiters). [#15199](https://github.com/ClickHouse/ClickHouse/pull/15199) ([filimonov](https://github.com/filimonov)). +* Add function `formatReadableTimeDelta` that format time delta to human readable string ... [#15497](https://github.com/ClickHouse/ClickHouse/pull/15497) ([Filipe Caixeta](https://github.com/filipecaixeta)). +* Add `tid` and `logTrace` function. This closes [#9434](https://github.com/ClickHouse/ClickHouse/issues/9434). [#15803](https://github.com/ClickHouse/ClickHouse/pull/15803) ([flynn](https://github.com/ucasfl)). +* Add a new option `print_query_id` to clickhouse-client. It helps generate arbitrary strings with the current query id generated by the client. [#15809](https://github.com/ClickHouse/ClickHouse/pull/15809) ([Amos Bird](https://github.com/amosbird)). +* Allow specify primary key in column list of CREATE TABLE query. [#15823](https://github.com/ClickHouse/ClickHouse/pull/15823) ([Maksim Kita](https://github.com/kitaisreal)). +* Added setting date_time_output_format. [#15845](https://github.com/ClickHouse/ClickHouse/pull/15845) ([Maksim Kita](https://github.com/kitaisreal)). +* Implement ``` OFFSET offset_row_count {ROW | ROWS} FETCH {FIRST | NEXT} fetch_row_count {ROW | ROWS} {ONLY | WITH TIES} ``` in select Query with order by. related issue:[#15367](https://github.com/ClickHouse/ClickHouse/issues/15367). [#15855](https://github.com/ClickHouse/ClickHouse/pull/15855) ([hexiaoting](https://github.com/hexiaoting)). +* Added an aggregate function, which calculates the p-value used for Welch's t-test. [#15874](https://github.com/ClickHouse/ClickHouse/pull/15874) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add max_concurrent_queries_for_all_users setting, see [#6636](https://github.com/ClickHouse/ClickHouse/issues/6636) for use cases. [#16154](https://github.com/ClickHouse/ClickHouse/pull/16154) ([nvartolomei](https://github.com/nvartolomei)). +* Added minimal web UI to ClickHouse. [#16158](https://github.com/ClickHouse/ClickHouse/pull/16158) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added function `untuple` which is a special function which can introduce new columns to the SELECT list by flattening a named tuple. [#16242](https://github.com/ClickHouse/ClickHouse/pull/16242) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Added toUUIDOrNull, toUUIDOrZero cast functions. [#16337](https://github.com/ClickHouse/ClickHouse/pull/16337) ([Maksim Kita](https://github.com/kitaisreal)). +* Add `system.replicated_fetches` table which shows currently running background fetches. [#16428](https://github.com/ClickHouse/ClickHouse/pull/16428) ([alesapin](https://github.com/alesapin)). +* - `errorCodeToName()` function - return variable name of the error (useful for analyzing query_log and similar) - `system.errors` table - shows how many times errors has been happened (respects `system_events_show_zero_values`). [#16438](https://github.com/ClickHouse/ClickHouse/pull/16438) ([Azat Khuzhin](https://github.com/azat)). +* Ability to create a docker image on the top of alpine. Uses precompiled binary and glibc components from ubuntu 20.04. [#16479](https://github.com/ClickHouse/ClickHouse/pull/16479) ([filimonov](https://github.com/filimonov)). +* Add `log_queries_min_query_duration_ms`, only queries slower then the value of this setting will go to `query_log`/`query_thread_log` (i.e. something like `slow_query_log` in mysql). [#16529](https://github.com/ClickHouse/ClickHouse/pull/16529) ([Azat Khuzhin](https://github.com/azat)). +* > Add farmFingerprint64 function. [#16570](https://github.com/ClickHouse/ClickHouse/pull/16570) ([Jacob Hayes](https://github.com/JacobHayes)). +* Now we can provide identifiers via query parameters. And these parameters can be used as table objects or columns. [#3815](https://github.com/ClickHouse/ClickHouse/issues/3815). [#16594](https://github.com/ClickHouse/ClickHouse/pull/16594) ([Amos Bird](https://github.com/amosbird)). +* Added big integers (UInt256, Int128, Int256) and UUID data types support for MergeTree BloomFilter index. [#16642](https://github.com/ClickHouse/ClickHouse/pull/16642) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Performance Improvement +* Speedup hashed/sparse_hashed dictionary loading by preallocating the hash table. [#15454](https://github.com/ClickHouse/ClickHouse/pull/15454) ([Azat Khuzhin](https://github.com/azat)). +* Do not merge parts across partitions in SELECT FINAL. [#15938](https://github.com/ClickHouse/ClickHouse/pull/15938) ([Kruglov Pavel](https://github.com/Avogar)). +* Improved performance of merges assignment in MergeTree table engines. Shouldn't be visible for the user. [#16191](https://github.com/ClickHouse/ClickHouse/pull/16191) ([alesapin](https://github.com/alesapin)). +* Improve performance of logical functions a little. [#16347](https://github.com/ClickHouse/ClickHouse/pull/16347) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve performance of `quantileMerge`. In previous versions it was obnoxiously slow. This closes [#1463](https://github.com/ClickHouse/ClickHouse/issues/1463). [#16643](https://github.com/ClickHouse/ClickHouse/pull/16643) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve performance of `-OrNull` and `-OrDefault` aggregate functions. [#16661](https://github.com/ClickHouse/ClickHouse/pull/16661) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Allow explicitly specify columns list in `CREATE TABLE table AS table_function(...)` query. Fixes [#9249](https://github.com/ClickHouse/ClickHouse/issues/9249) Fixes [#14214](https://github.com/ClickHouse/ClickHouse/issues/14214). [#14295](https://github.com/ClickHouse/ClickHouse/pull/14295) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Now trivial count optimization becomes slightly non-trivial. Predicates that contain exact partition expr can be optimized too. This also fixes [#11092](https://github.com/ClickHouse/ClickHouse/issues/11092) which returns wrong count when `max_parallel_replicas > 1`. [#15074](https://github.com/ClickHouse/ClickHouse/pull/15074) ([Amos Bird](https://github.com/amosbird)). +* Enable parsing enum values by their ids for CSV, TSV and JSON input formats. [#15685](https://github.com/ClickHouse/ClickHouse/pull/15685) ([vivarum](https://github.com/vivarum)). +* Add reconnects to `zookeeper-dump-tree` tool. [#15711](https://github.com/ClickHouse/ClickHouse/pull/15711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove `MemoryTrackingInBackground*` metrics to avoid potentially misleading results. This fixes [#15684](https://github.com/ClickHouse/ClickHouse/issues/15684). [#15813](https://github.com/ClickHouse/ClickHouse/pull/15813) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change level of some log messages from information to debug, so information messages will not appear for every query. This closes [#5293](https://github.com/ClickHouse/ClickHouse/issues/5293). [#15816](https://github.com/ClickHouse/ClickHouse/pull/15816) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix query hang (endless loop) in case of misconfiguration (`connections_with_failover_max_tries` set to 0). [#15876](https://github.com/ClickHouse/ClickHouse/pull/15876) ([Azat Khuzhin](https://github.com/azat)). +* Added boost::program_options to `db_generator` in order to increase its usability. This closes [#15940](https://github.com/ClickHouse/ClickHouse/issues/15940). [#15973](https://github.com/ClickHouse/ClickHouse/pull/15973) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Treat `INTERVAL '1 hour'` as equivalent to `INTERVAL 1 HOUR`, to be compatible with Postgres. This fixes [#15637](https://github.com/ClickHouse/ClickHouse/issues/15637). [#15978](https://github.com/ClickHouse/ClickHouse/pull/15978) ([flynn](https://github.com/ucasfl)). +* Simplify the implementation of background tasks processing for the MergeTree table engines family. There should be no visible changes for user. [#15983](https://github.com/ClickHouse/ClickHouse/pull/15983) ([alesapin](https://github.com/alesapin)). +* Add support of cache layout for Redis dictionaries with complex key. [#15985](https://github.com/ClickHouse/ClickHouse/pull/15985) ([Anton Popov](https://github.com/CurtizJ)). +* Fix rare issue when clickhouse-client may abort on exit due to loading of suggestions. This fixes [#16035](https://github.com/ClickHouse/ClickHouse/issues/16035). [#16047](https://github.com/ClickHouse/ClickHouse/pull/16047) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now it's allowed to execute `ALTER ... ON CLUSTER` queries regardless of the `` setting in cluster config. [#16075](https://github.com/ClickHouse/ClickHouse/pull/16075) ([alesapin](https://github.com/alesapin)). +* - Fix memory_profiler_step/max_untracked_memory for queries via HTTP (test included), and adjusting this value globally in xml config will not help either, since those settings are not applied anyway, only default (4MB) value is [used](https://github.com/ClickHouse/ClickHouse/blob/17731245336d8c84f75e4c0894c5797ed7732190/src/Common/ThreadStatus.h#L104). - Fix query_id for the most root ThreadStatus of the http query (by initializing QueryScope after reading query_id). [#16101](https://github.com/ClickHouse/ClickHouse/pull/16101) ([Azat Khuzhin](https://github.com/azat)). +* Add allow_nondeterministic_optimize_skip_unused_shards (to allow non deterministic like rand() or dictGet() in sharding key). [#16105](https://github.com/ClickHouse/ClickHouse/pull/16105) ([Azat Khuzhin](https://github.com/azat)). +* database_atomic_wait_for_drop_and_detach_synchronously/NO DELAY/SYNC for DROP DATABASE. [#16127](https://github.com/ClickHouse/ClickHouse/pull/16127) ([Azat Khuzhin](https://github.com/azat)). +* Add support for nested data types (like named tuple) as sub-types. Fixes [#15587](https://github.com/ClickHouse/ClickHouse/issues/15587). [#16262](https://github.com/ClickHouse/ClickHouse/pull/16262) ([Ivan](https://github.com/abyss7)). +* If there are no tmp folder in the system (chroot, misconfigutation etc) clickhouse-local will create temporary subfolder in the current directory. [#16280](https://github.com/ClickHouse/ClickHouse/pull/16280) ([filimonov](https://github.com/filimonov)). +* Now it's possible to specify `PRIMARY KEY` without `ORDER BY` for MergeTree table engines family. Closes [#15591](https://github.com/ClickHouse/ClickHouse/issues/15591). [#16284](https://github.com/ClickHouse/ClickHouse/pull/16284) ([alesapin](https://github.com/alesapin)). +* try use cmake version for croaring instead of amalgamation.sh. [#16285](https://github.com/ClickHouse/ClickHouse/pull/16285) ([sundyli](https://github.com/sundy-li)). +* Add total_rows/total_bytes (from system.tables) support for Set/Join table engines. [#16306](https://github.com/ClickHouse/ClickHouse/pull/16306) ([Azat Khuzhin](https://github.com/azat)). +* Better diagnostics when client has dropped connection. In previous versions, `Attempt to read after EOF` and `Broken pipe` exceptions were logged in server. In new version, it's information message `Client has dropped the connection, cancel the query.`. [#16329](https://github.com/ClickHouse/ClickHouse/pull/16329) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `TablesToDropQueueSize` metric. It's equal to number of dropped tables, that are waiting for background data removal. [#16364](https://github.com/ClickHouse/ClickHouse/pull/16364) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix debug assertion in quantileDeterministic function. In previous version it may also transfer up to two times more data over the network. Although no bug existed. This fixes [#15683](https://github.com/ClickHouse/ClickHouse/issues/15683). [#16410](https://github.com/ClickHouse/ClickHouse/pull/16410) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better read task scheduling for JBOD architecture and `MergeTree` storage. New setting `read_backoff_min_concurrency` which serves as the lower limit to the number of reading threads. [#16423](https://github.com/ClickHouse/ClickHouse/pull/16423) ([Amos Bird](https://github.com/amosbird)). +* Fixed bug for [#16263](https://github.com/ClickHouse/ClickHouse/issues/16263). Also minimized event loop lifetime. Added more efficient queues setup. [#16426](https://github.com/ClickHouse/ClickHouse/pull/16426) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Allow to fetch parts that are already committed or outdated in the current instance into the detached directory. It's useful when migrating tables from another cluster and having N to 1 shards mapping. It's also consistent with the current fetchPartition implementation. [#16538](https://github.com/ClickHouse/ClickHouse/pull/16538) ([Amos Bird](https://github.com/amosbird)). +* Add current_database into query_thread_log. [#16558](https://github.com/ClickHouse/ClickHouse/pull/16558) ([Azat Khuzhin](https://github.com/azat)). +* Subqueries in WITH section (CTE) can reference previous subqueries in WITH section by their name. [#16575](https://github.com/ClickHouse/ClickHouse/pull/16575) ([Amos Bird](https://github.com/amosbird)). +* - Improve scheduling of background task which removes data of dropped tables in `Atomic` databases. - `Atomic` databases do not create broken symlink to table data directory if table actually has no data directory. [#16584](https://github.com/ClickHouse/ClickHouse/pull/16584) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Now paratmeterized functions can be used in APPLY column transformer. [#16589](https://github.com/ClickHouse/ClickHouse/pull/16589) ([Amos Bird](https://github.com/amosbird)). +* Now `event_time_microseconds` field stores in Decimal64, not UInt64. Removed an incorrect check from Field::get(). [#16617](https://github.com/ClickHouse/ClickHouse/pull/16617) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Apply SETTINGS clause as early as possible. It allows to modify more settings in the query. This closes [#3178](https://github.com/ClickHouse/ClickHouse/issues/3178). [#16619](https://github.com/ClickHouse/ClickHouse/pull/16619) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better update of ZooKeeper configuration in runtime. [#16630](https://github.com/ClickHouse/ClickHouse/pull/16630) ([sundyli](https://github.com/sundy-li)). +* Make the behaviour of `minMap` and `maxMap` more desireable. It will not skip zero values in the result. Fixes [#16087](https://github.com/ClickHouse/ClickHouse/issues/16087). [#16631](https://github.com/ClickHouse/ClickHouse/pull/16631) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Better diagnostics on parse errors in input data. Provide row number on `Cannot read all data` errors. [#16644](https://github.com/ClickHouse/ClickHouse/pull/16644) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Update jemalloc to fix percpu_arena with affinity mask. [#15035](https://github.com/ClickHouse/ClickHouse/pull/15035) ([Azat Khuzhin](https://github.com/azat)). +* Decrement the `ReadonlyReplica` metric when detaching read-only tables. This fixes [#15598](https://github.com/ClickHouse/ClickHouse/issues/15598). [#15592](https://github.com/ClickHouse/ClickHouse/pull/15592) ([sundyli](https://github.com/sundy-li)). +* Fixed bug with globs in S3 table function, region from URL was not applied to S3 client configuration. [#15646](https://github.com/ClickHouse/ClickHouse/pull/15646) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix error `Cannot add simple transform to empty Pipe` which happened while reading from `Buffer` table which has different structure than destination table. It was possible if destination table returned empty result for query. Fixes [#15529](https://github.com/ClickHouse/ClickHouse/issues/15529). [#15662](https://github.com/ClickHouse/ClickHouse/pull/15662) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Possibility to move part to another disk/volume if the first attempt was failed. [#15723](https://github.com/ClickHouse/ClickHouse/pull/15723) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix drop of materialized view with inner table in Atomic database (hangs all subsequent DROP TABLE due to hang of the worker thread, due to recursive DROP TABLE for inner table of MV). [#15743](https://github.com/ClickHouse/ClickHouse/pull/15743) ([Azat Khuzhin](https://github.com/azat)). +* Fix `select count()` inaccuracy for MaterializeMySQL. [#15767](https://github.com/ClickHouse/ClickHouse/pull/15767) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix exception `Block structure mismatch` in `SELECT ... ORDER BY DESC` queries which were executed after `ALTER MODIFY COLUMN` query. Fixes [#15800](https://github.com/ClickHouse/ClickHouse/issues/15800). [#15852](https://github.com/ClickHouse/ClickHouse/pull/15852) ([alesapin](https://github.com/alesapin)). +* Now exception will be thrown when `ALTER MODIFY COLUMN ... DEFAULT ...` has incompatible default with column type. Fixes [#15854](https://github.com/ClickHouse/ClickHouse/issues/15854). [#15858](https://github.com/ClickHouse/ClickHouse/pull/15858) ([alesapin](https://github.com/alesapin)). +* Fix possible deadlocks in RBAC. [#15875](https://github.com/ClickHouse/ClickHouse/pull/15875) ([Vitaly Baranov](https://github.com/vitlibar)). +* fixes [#12513](https://github.com/ClickHouse/ClickHouse/issues/12513) fix difference expressions with same alias when analyze queries again. [#15886](https://github.com/ClickHouse/ClickHouse/pull/15886) ([Winter Zhang](https://github.com/zhang2014)). +* Fix incorrect empty result for query from `Distributed` table if query has `WHERE`, `PREWHERE` and `GLOBAL IN`. Fixes [#15792](https://github.com/ClickHouse/ClickHouse/issues/15792). [#15933](https://github.com/ClickHouse/ClickHouse/pull/15933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed `DROP TABLE IF EXISTS` failure with `Table ... doesn't exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) Fixed `DROP/DETACH DATABASE` failure with `Table ... doesn't exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix a crash when database creation fails. [#15954](https://github.com/ClickHouse/ClickHouse/pull/15954) ([Winter Zhang](https://github.com/zhang2014)). +* Fix ambiguity in parsing of settings profiles: `CREATE USER ... SETTINGS profile readonly` is now considered as using a profile named `readonly`, not a setting named `profile` with the readonly constraint. This fixes [#15628](https://github.com/ClickHouse/ClickHouse/issues/15628). [#15982](https://github.com/ClickHouse/ClickHouse/pull/15982) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix rare segfaults when inserting into or selecting from MaterializedView and concurrently dropping target table (for Atomic database engine). [#15984](https://github.com/ClickHouse/ClickHouse/pull/15984) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Prevent replica hang for 5-10 mins when replication error happens after a period of inactivity. [#15987](https://github.com/ClickHouse/ClickHouse/pull/15987) ([filimonov](https://github.com/filimonov)). +* Allow to use direct layout for dictionaries with complex keys. [#16007](https://github.com/ClickHouse/ClickHouse/pull/16007) ([Anton Popov](https://github.com/CurtizJ)). +* Fix collate name & charset name parser and support `length = 0` for string type. [#16008](https://github.com/ClickHouse/ClickHouse/pull/16008) ([Winter Zhang](https://github.com/zhang2014)). +* Fix `ALTER MODIFY ... ORDER BY` query hang for `ReplicatedVersionedCollapsingMergeTree`. This fixes [#15980](https://github.com/ClickHouse/ClickHouse/issues/15980). [#16011](https://github.com/ClickHouse/ClickHouse/pull/16011) ([alesapin](https://github.com/alesapin)). +* Fix bug with MySQL database. When MySQL server used as database engine is down some queries raise Exception, because they try to get tables from disabled server, while it's unnecessary. For example, query `SELECT ... FROM system.parts` should work only with MergeTree tables and don't touch MySQL database at all. [#16032](https://github.com/ClickHouse/ClickHouse/pull/16032) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixes [#15780](https://github.com/ClickHouse/ClickHouse/issues/15780) regression, e.g. indexOf([1, 2, 3], toLowCardinality(1)) now is prohibited but it should not be. [#16038](https://github.com/ClickHouse/ClickHouse/pull/16038) ([Mike Kot](https://github.com/myrrc)). +* Fix segfault in some cases of wrong aggregation in lambdas. [#16082](https://github.com/ClickHouse/ClickHouse/pull/16082) ([Anton Popov](https://github.com/CurtizJ)). +* Fix the `clickhouse-local` crash when trying to do `OPTIMIZE` command. Fixes [#16076](https://github.com/ClickHouse/ClickHouse/issues/16076). [#16192](https://github.com/ClickHouse/ClickHouse/pull/16192) ([filimonov](https://github.com/filimonov)). +* Fix dictGet in sharding_key (and similar places, i.e. when the function context is stored permanently). [#16205](https://github.com/ClickHouse/ClickHouse/pull/16205) ([Azat Khuzhin](https://github.com/azat)). +* Fix the case when memory can be overallocated regardless to the limit. This closes [#14560](https://github.com/ClickHouse/ClickHouse/issues/14560). [#16206](https://github.com/ClickHouse/ClickHouse/pull/16206) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a possible memory leak during `GROUP BY` with string keys, caused by an error in `TwoLevelStringHashTable` implementation. [#16264](https://github.com/ClickHouse/ClickHouse/pull/16264) ([Amos Bird](https://github.com/amosbird)). +* Fixed the inconsistent behaviour when a part of return data could be dropped because the set for its filtration wasn't created. [#16308](https://github.com/ClickHouse/ClickHouse/pull/16308) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix processing of very large entries in replication queue. Very large entries may appear in ALTER queries if table structure is extremely large (near 1 MB). This fixes [#16307](https://github.com/ClickHouse/ClickHouse/issues/16307). [#16332](https://github.com/ClickHouse/ClickHouse/pull/16332) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix async Distributed INSERT w/ prefer_localhost_replica=0 and internal_replication. [#16358](https://github.com/ClickHouse/ClickHouse/pull/16358) ([Azat Khuzhin](https://github.com/azat)). +* Fix group by with totals/rollup/cube modifers and min/max functions over group by keys. Fixes [#16393](https://github.com/ClickHouse/ClickHouse/issues/16393). [#16397](https://github.com/ClickHouse/ClickHouse/pull/16397) ([Anton Popov](https://github.com/CurtizJ)). +* Fix DROP TABLE for Distributed (racy with INSERT). [#16409](https://github.com/ClickHouse/ClickHouse/pull/16409) ([Azat Khuzhin](https://github.com/azat)). +* Fix double free in case of exception in function `dictGet`. It could have happened if dictionary was loaded with error. [#16429](https://github.com/ClickHouse/ClickHouse/pull/16429) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Specifically crafter argument of `round` function with `Decimal` was leading to integer division by zero. This fixes [#13338](https://github.com/ClickHouse/ClickHouse/issues/13338). [#16451](https://github.com/ClickHouse/ClickHouse/pull/16451) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix rapid growth of metadata when using MySQL Master -> MySQL Slave -> ClickHouse MaterializeMySQL Engine, and `slave_parallel_worker` enabled on MySQL Slave, by properly shrinking GTID sets. This fixes [#15951](https://github.com/ClickHouse/ClickHouse/issues/15951). [#16504](https://github.com/ClickHouse/ClickHouse/pull/16504) ([TCeason](https://github.com/TCeason)). +* Now when parsing AVRO from input the LowCardinality is removed from type. Fixes [#16188](https://github.com/ClickHouse/ClickHouse/issues/16188). [#16521](https://github.com/ClickHouse/ClickHouse/pull/16521) ([Mike Kot](https://github.com/myrrc)). +* Fix query_thread_log.query_duration_ms unit. [#16563](https://github.com/ClickHouse/ClickHouse/pull/16563) ([Azat Khuzhin](https://github.com/azat)). +* Calculation of `DEFAULT` expressions was involving possible name collisions (that was very unlikely to encounter). This fixes [#9359](https://github.com/ClickHouse/ClickHouse/issues/9359). [#16612](https://github.com/ClickHouse/ClickHouse/pull/16612) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The setting `max_parallel_replicas` worked incorrectly if the queried table has no sampling. This fixes [#5733](https://github.com/ClickHouse/ClickHouse/issues/5733). [#16675](https://github.com/ClickHouse/ClickHouse/pull/16675) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Build/Testing/Packaging Improvement +* Simplify Sys/V init script. [#14135](https://github.com/ClickHouse/ClickHouse/pull/14135) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix illegal code style `&vector[idx]` in libhdfs3. This fixes libcxx debug build. See also https://github.com/ClickHouse-Extras/libhdfs3/pull/8 . [#15815](https://github.com/ClickHouse/ClickHouse/pull/15815) ([Amos Bird](https://github.com/amosbird)). +* Check for `#pragma once` in headers. [#15818](https://github.com/ClickHouse/ClickHouse/pull/15818) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Check for executable bit on non-executable files. People often accidentially commit executable files from Windows. [#15843](https://github.com/ClickHouse/ClickHouse/pull/15843) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* - Updated RBAC syntax tests. - Updated `INSERT` and `SELECT` RBAC tests. - New RBAC tests for `ALTER` privileges by @ritaank - New RBAC tests for `MATERIALIZED`, `LIVE`, and standard `VIEWS`. - New RBAC tests for public system tables. - New RBAC tests for `SHOW TABLES`. [#16044](https://github.com/ClickHouse/ClickHouse/pull/16044) ([MyroTk](https://github.com/MyroTk)). +* Refuse to build with AppleClang because it's difficult to find out version correspondence. This closes [#16072](https://github.com/ClickHouse/ClickHouse/issues/16072). [#16074](https://github.com/ClickHouse/ClickHouse/pull/16074) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add scipy to fasttest docker. [#16093](https://github.com/ClickHouse/ClickHouse/pull/16093) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add MySQL BinLog file check tool for MaterializeMySQL database engine. [#16223](https://github.com/ClickHouse/ClickHouse/pull/16223) ([Winter Zhang](https://github.com/zhang2014)). +* Add flaky check for stateless tests. [#16238](https://github.com/ClickHouse/ClickHouse/pull/16238) ([alesapin](https://github.com/alesapin)). +* Converting test tests/queries/0_stateless/01446_json_strings_each_row to a shell script. [#16247](https://github.com/ClickHouse/ClickHouse/pull/16247) ([vzakaznikov](https://github.com/vzakaznikov)). +* - None. [#16249](https://github.com/ClickHouse/ClickHouse/pull/16249) ([Denis Glazachev](https://github.com/traceon)). +* Fixing fails in LDAP external user directory tests. [#16363](https://github.com/ClickHouse/ClickHouse/pull/16363) ([vzakaznikov](https://github.com/vzakaznikov)). +* During config removal wrong exit code was expected. [#16365](https://github.com/ClickHouse/ClickHouse/pull/16365) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fix LDAP tests by grabbing log size after container is stopped. [#16440](https://github.com/ClickHouse/ClickHouse/pull/16440) ([vzakaznikov](https://github.com/vzakaznikov)). +* Improve generation of build files for `ya.make` build system (Arcadia). [#16700](https://github.com/ClickHouse/ClickHouse/pull/16700) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Other +* Use only |name_parts| as primary name source and auto-generate full name. [#16149](https://github.com/ClickHouse/ClickHouse/pull/16149) ([Ivan](https://github.com/abyss7)). +* Rename struct NullSink from ReadHelpers to NullOutput, because class NullSink exists in Processors/NullSink.h. It's needed to prevent redefinition of 'NullSink' error. [#16520](https://github.com/ClickHouse/ClickHouse/pull/16520) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NO CL CATEGORY + +* Try to make MergeTreeWriteAheadLog forward compatible. [#16094](https://github.com/ClickHouse/ClickHouse/pull/16094) ([nvartolomei](https://github.com/nvartolomei)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Write structure of table functions to metadata"'. [#15961](https://github.com/ClickHouse/ClickHouse/pull/15961) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "scipy"'. [#16156](https://github.com/ClickHouse/ClickHouse/pull/16156) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* NO CL ENTRY: 'Bump markdown from 3.2.1 to 3.3.2 in /docs/tools'. [#16180](https://github.com/ClickHouse/ClickHouse/pull/16180) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Revert "Optionally upload clickhouse binary in fast test"'. [#16333](https://github.com/ClickHouse/ClickHouse/pull/16333) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'minor fix.'. [#16335](https://github.com/ClickHouse/ClickHouse/pull/16335) ([Xianda Ke](https://github.com/kexianda)). +* NO CL ENTRY: 'Bump tornado from 5.1.1 to 6.1 in /docs/tools'. [#16590](https://github.com/ClickHouse/ClickHouse/pull/16590) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump mkdocs-macros-plugin from 0.4.17 to 0.4.20 in /docs/tools'. [#16692](https://github.com/ClickHouse/ClickHouse/pull/16692) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). + diff --git a/docs/changelogs/v20.11.2.1-stable.md b/docs/changelogs/v20.11.2.1-stable.md new file mode 100644 index 00000000000..b3a71a0fdfa --- /dev/null +++ b/docs/changelogs/v20.11.2.1-stable.md @@ -0,0 +1,162 @@ +### ClickHouse release v20.11.2.1-stable FIXME as compared to v20.10.1.4881-prestable + +#### Backward Incompatible Change +* Make rankCorr function return nan on insufficient data [#16124](https://github.com/ClickHouse/ClickHouse/issues/16124). [#16135](https://github.com/ClickHouse/ClickHouse/pull/16135) ([hexiaoting](https://github.com/hexiaoting)). +* Aggregate functions `boundingRatio`, `rankCorr`, `retention`, `timeSeriesGroupSum`, `timeSeriesGroupRateSum`, `windowFunnel` were erroneously made case-insensitive. Now their names are made case sensitive as designed. Only functions that are specified in SQL standard or made for compatibility with other DBMS or functions similar to those should be case-insensitive. [#16407](https://github.com/ClickHouse/ClickHouse/pull/16407) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove `ANALYZE` and `AST` queries, and make the setting `enable_debug_queries` obsolete since now it is the part of full featured `EXPLAIN` query. [#16536](https://github.com/ClickHouse/ClickHouse/pull/16536) ([Ivan](https://github.com/abyss7)). +* Restrict to use of non-comparable data types (like `AggregateFunction`) in keys (Sorting key, Primary key, Partition key, and so on). [#16601](https://github.com/ClickHouse/ClickHouse/pull/16601) ([alesapin](https://github.com/alesapin)). +* If some `profile` was specified in `distributed_ddl` config section, then this profile could overwrite settings of `default` profile on server startup. It's fixed, now settings of distributed DDL queries should not affect global server settings. [#16635](https://github.com/ClickHouse/ClickHouse/pull/16635) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### New Feature +* #WelchTTest aggregate function implementation. [#10351](https://github.com/ClickHouse/ClickHouse/pull/10351) ([antikvist](https://github.com/antikvist)). +* New functions `encrypt`, `aes_encrypt_mysql`, `decrypt`, `aes_decrypt_mysql`. These functions are working slowly (below ClickHouse standards), so we consider it as an experimental feature. [#11844](https://github.com/ClickHouse/ClickHouse/pull/11844) ([Vasily Nemkov](https://github.com/Enmk)). +* - Added support of LDAP as a user directory for locally non-existent users. [#12736](https://github.com/ClickHouse/ClickHouse/pull/12736) ([Denis Glazachev](https://github.com/traceon)). +* Added `disable_merges` option for volumes in multi-disk configuration. [#13956](https://github.com/ClickHouse/ClickHouse/pull/13956) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Added initial OpenTelemetry support. ClickHouse now accepts OpenTelemetry traceparent headers over Native and HTTP protocols, and passes them downstream in some cases. The trace spans for executed queries are saved into the `system.opentelemetry_span_log` table. [#14195](https://github.com/ClickHouse/ClickHouse/pull/14195) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allows to read/write Single protobuf message at once (w/o length-delimiters). [#15199](https://github.com/ClickHouse/ClickHouse/pull/15199) ([filimonov](https://github.com/filimonov)). +* Add function `formatReadableTimeDelta` that format time delta to human readable string ... [#15497](https://github.com/ClickHouse/ClickHouse/pull/15497) ([Filipe Caixeta](https://github.com/filipecaixeta)). +* Add `tid` and `logTrace` function. This closes [#9434](https://github.com/ClickHouse/ClickHouse/issues/9434). [#15803](https://github.com/ClickHouse/ClickHouse/pull/15803) ([flynn](https://github.com/ucasfl)). +* Add a new option `print_query_id` to clickhouse-client. It helps generate arbitrary strings with the current query id generated by the client. [#15809](https://github.com/ClickHouse/ClickHouse/pull/15809) ([Amos Bird](https://github.com/amosbird)). +* Allow specify primary key in column list of CREATE TABLE query. [#15823](https://github.com/ClickHouse/ClickHouse/pull/15823) ([Maksim Kita](https://github.com/kitaisreal)). +* Added setting date_time_output_format. [#15845](https://github.com/ClickHouse/ClickHouse/pull/15845) ([Maksim Kita](https://github.com/kitaisreal)). +* Implement ``` OFFSET offset_row_count {ROW | ROWS} FETCH {FIRST | NEXT} fetch_row_count {ROW | ROWS} {ONLY | WITH TIES} ``` in select Query with order by. related issue:[#15367](https://github.com/ClickHouse/ClickHouse/issues/15367). [#15855](https://github.com/ClickHouse/ClickHouse/pull/15855) ([hexiaoting](https://github.com/hexiaoting)). +* Added an aggregate function, which calculates the p-value used for Welch's t-test. [#15874](https://github.com/ClickHouse/ClickHouse/pull/15874) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add max_concurrent_queries_for_all_users setting, see [#6636](https://github.com/ClickHouse/ClickHouse/issues/6636) for use cases. [#16154](https://github.com/ClickHouse/ClickHouse/pull/16154) ([nvartolomei](https://github.com/nvartolomei)). +* Added minimal web UI to ClickHouse. [#16158](https://github.com/ClickHouse/ClickHouse/pull/16158) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added function `untuple` which is a special function which can introduce new columns to the SELECT list by flattening a named tuple. [#16242](https://github.com/ClickHouse/ClickHouse/pull/16242) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Added toUUIDOrNull, toUUIDOrZero cast functions. [#16337](https://github.com/ClickHouse/ClickHouse/pull/16337) ([Maksim Kita](https://github.com/kitaisreal)). +* Add `system.replicated_fetches` table which shows currently running background fetches. [#16428](https://github.com/ClickHouse/ClickHouse/pull/16428) ([alesapin](https://github.com/alesapin)). +* - `errorCodeToName()` function - return variable name of the error (useful for analyzing query_log and similar) - `system.errors` table - shows how many times errors has been happened (respects `system_events_show_zero_values`). [#16438](https://github.com/ClickHouse/ClickHouse/pull/16438) ([Azat Khuzhin](https://github.com/azat)). +* Ability to create a docker image on the top of alpine. Uses precompiled binary and glibc components from ubuntu 20.04. [#16479](https://github.com/ClickHouse/ClickHouse/pull/16479) ([filimonov](https://github.com/filimonov)). +* Add `log_queries_min_query_duration_ms`, only queries slower then the value of this setting will go to `query_log`/`query_thread_log` (i.e. something like `slow_query_log` in mysql). [#16529](https://github.com/ClickHouse/ClickHouse/pull/16529) ([Azat Khuzhin](https://github.com/azat)). +* > Add farmFingerprint64 function. [#16570](https://github.com/ClickHouse/ClickHouse/pull/16570) ([Jacob Hayes](https://github.com/JacobHayes)). +* Now we can provide identifiers via query parameters. And these parameters can be used as table objects or columns. [#3815](https://github.com/ClickHouse/ClickHouse/issues/3815). [#16594](https://github.com/ClickHouse/ClickHouse/pull/16594) ([Amos Bird](https://github.com/amosbird)). +* Added big integers (UInt256, Int128, Int256) and UUID data types support for MergeTree BloomFilter index. [#16642](https://github.com/ClickHouse/ClickHouse/pull/16642) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Performance Improvement +* Speedup hashed/sparse_hashed dictionary loading by preallocating the hash table. [#15454](https://github.com/ClickHouse/ClickHouse/pull/15454) ([Azat Khuzhin](https://github.com/azat)). +* Do not merge parts across partitions in SELECT FINAL. [#15938](https://github.com/ClickHouse/ClickHouse/pull/15938) ([Kruglov Pavel](https://github.com/Avogar)). +* Improved performance of merges assignment in MergeTree table engines. Shouldn't be visible for the user. [#16191](https://github.com/ClickHouse/ClickHouse/pull/16191) ([alesapin](https://github.com/alesapin)). +* Improve performance of logical functions a little. [#16347](https://github.com/ClickHouse/ClickHouse/pull/16347) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve performance of `quantileMerge`. In previous versions it was obnoxiously slow. This closes [#1463](https://github.com/ClickHouse/ClickHouse/issues/1463). [#16643](https://github.com/ClickHouse/ClickHouse/pull/16643) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve performance of `-OrNull` and `-OrDefault` aggregate functions. [#16661](https://github.com/ClickHouse/ClickHouse/pull/16661) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Allow explicitly specify columns list in `CREATE TABLE table AS table_function(...)` query. Fixes [#9249](https://github.com/ClickHouse/ClickHouse/issues/9249) Fixes [#14214](https://github.com/ClickHouse/ClickHouse/issues/14214). [#14295](https://github.com/ClickHouse/ClickHouse/pull/14295) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Now trivial count optimization becomes slightly non-trivial. Predicates that contain exact partition expr can be optimized too. This also fixes [#11092](https://github.com/ClickHouse/ClickHouse/issues/11092) which returns wrong count when `max_parallel_replicas > 1`. [#15074](https://github.com/ClickHouse/ClickHouse/pull/15074) ([Amos Bird](https://github.com/amosbird)). +* Enable parsing enum values by their ids for CSV, TSV and JSON input formats. [#15685](https://github.com/ClickHouse/ClickHouse/pull/15685) ([vivarum](https://github.com/vivarum)). +* Add reconnects to `zookeeper-dump-tree` tool. [#15711](https://github.com/ClickHouse/ClickHouse/pull/15711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove `MemoryTrackingInBackground*` metrics to avoid potentially misleading results. This fixes [#15684](https://github.com/ClickHouse/ClickHouse/issues/15684). [#15813](https://github.com/ClickHouse/ClickHouse/pull/15813) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change level of some log messages from information to debug, so information messages will not appear for every query. This closes [#5293](https://github.com/ClickHouse/ClickHouse/issues/5293). [#15816](https://github.com/ClickHouse/ClickHouse/pull/15816) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix query hang (endless loop) in case of misconfiguration (`connections_with_failover_max_tries` set to 0). [#15876](https://github.com/ClickHouse/ClickHouse/pull/15876) ([Azat Khuzhin](https://github.com/azat)). +* Added boost::program_options to `db_generator` in order to increase its usability. This closes [#15940](https://github.com/ClickHouse/ClickHouse/issues/15940). [#15973](https://github.com/ClickHouse/ClickHouse/pull/15973) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Treat `INTERVAL '1 hour'` as equivalent to `INTERVAL 1 HOUR`, to be compatible with Postgres. This fixes [#15637](https://github.com/ClickHouse/ClickHouse/issues/15637). [#15978](https://github.com/ClickHouse/ClickHouse/pull/15978) ([flynn](https://github.com/ucasfl)). +* Add support of cache layout for Redis dictionaries with complex key. [#15985](https://github.com/ClickHouse/ClickHouse/pull/15985) ([Anton Popov](https://github.com/CurtizJ)). +* Fix rare issue when clickhouse-client may abort on exit due to loading of suggestions. This fixes [#16035](https://github.com/ClickHouse/ClickHouse/issues/16035). [#16047](https://github.com/ClickHouse/ClickHouse/pull/16047) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now it's allowed to execute `ALTER ... ON CLUSTER` queries regardless of the `` setting in cluster config. [#16075](https://github.com/ClickHouse/ClickHouse/pull/16075) ([alesapin](https://github.com/alesapin)). +* - Fix memory_profiler_step/max_untracked_memory for queries via HTTP (test included), and adjusting this value globally in xml config will not help either, since those settings are not applied anyway, only default (4MB) value is [used](https://github.com/ClickHouse/ClickHouse/blob/17731245336d8c84f75e4c0894c5797ed7732190/src/Common/ThreadStatus.h#L104). - Fix query_id for the most root ThreadStatus of the http query (by initializing QueryScope after reading query_id). [#16101](https://github.com/ClickHouse/ClickHouse/pull/16101) ([Azat Khuzhin](https://github.com/azat)). +* Add allow_nondeterministic_optimize_skip_unused_shards (to allow non deterministic like rand() or dictGet() in sharding key). [#16105](https://github.com/ClickHouse/ClickHouse/pull/16105) ([Azat Khuzhin](https://github.com/azat)). +* database_atomic_wait_for_drop_and_detach_synchronously/NO DELAY/SYNC for DROP DATABASE. [#16127](https://github.com/ClickHouse/ClickHouse/pull/16127) ([Azat Khuzhin](https://github.com/azat)). +* Add support for nested data types (like named tuple) as sub-types. Fixes [#15587](https://github.com/ClickHouse/ClickHouse/issues/15587). [#16262](https://github.com/ClickHouse/ClickHouse/pull/16262) ([Ivan](https://github.com/abyss7)). +* If there are no tmp folder in the system (chroot, misconfigutation etc) clickhouse-local will create temporary subfolder in the current directory. [#16280](https://github.com/ClickHouse/ClickHouse/pull/16280) ([filimonov](https://github.com/filimonov)). +* Now it's possible to specify `PRIMARY KEY` without `ORDER BY` for MergeTree table engines family. Closes [#15591](https://github.com/ClickHouse/ClickHouse/issues/15591). [#16284](https://github.com/ClickHouse/ClickHouse/pull/16284) ([alesapin](https://github.com/alesapin)). +* try use cmake version for croaring instead of amalgamation.sh. [#16285](https://github.com/ClickHouse/ClickHouse/pull/16285) ([sundyli](https://github.com/sundy-li)). +* Add total_rows/total_bytes (from system.tables) support for Set/Join table engines. [#16306](https://github.com/ClickHouse/ClickHouse/pull/16306) ([Azat Khuzhin](https://github.com/azat)). +* Better diagnostics when client has dropped connection. In previous versions, `Attempt to read after EOF` and `Broken pipe` exceptions were logged in server. In new version, it's information message `Client has dropped the connection, cancel the query.`. [#16329](https://github.com/ClickHouse/ClickHouse/pull/16329) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `TablesToDropQueueSize` metric. It's equal to number of dropped tables, that are waiting for background data removal. [#16364](https://github.com/ClickHouse/ClickHouse/pull/16364) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix debug assertion in quantileDeterministic function. In previous version it may also transfer up to two times more data over the network. Although no bug existed. This fixes [#15683](https://github.com/ClickHouse/ClickHouse/issues/15683). [#16410](https://github.com/ClickHouse/ClickHouse/pull/16410) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better read task scheduling for JBOD architecture and `MergeTree` storage. New setting `read_backoff_min_concurrency` which serves as the lower limit to the number of reading threads. [#16423](https://github.com/ClickHouse/ClickHouse/pull/16423) ([Amos Bird](https://github.com/amosbird)). +* Fixed bug for [#16263](https://github.com/ClickHouse/ClickHouse/issues/16263). Also minimized event loop lifetime. Added more efficient queues setup. [#16426](https://github.com/ClickHouse/ClickHouse/pull/16426) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Allow to fetch parts that are already committed or outdated in the current instance into the detached directory. It's useful when migrating tables from another cluster and having N to 1 shards mapping. It's also consistent with the current fetchPartition implementation. [#16538](https://github.com/ClickHouse/ClickHouse/pull/16538) ([Amos Bird](https://github.com/amosbird)). +* Add current_database into query_thread_log. [#16558](https://github.com/ClickHouse/ClickHouse/pull/16558) ([Azat Khuzhin](https://github.com/azat)). +* Subqueries in WITH section (CTE) can reference previous subqueries in WITH section by their name. [#16575](https://github.com/ClickHouse/ClickHouse/pull/16575) ([Amos Bird](https://github.com/amosbird)). +* - Improve scheduling of background task which removes data of dropped tables in `Atomic` databases. - `Atomic` databases do not create broken symlink to table data directory if table actually has no data directory. [#16584](https://github.com/ClickHouse/ClickHouse/pull/16584) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Now paratmeterized functions can be used in APPLY column transformer. [#16589](https://github.com/ClickHouse/ClickHouse/pull/16589) ([Amos Bird](https://github.com/amosbird)). +* Now `event_time_microseconds` field stores in Decimal64, not UInt64. Removed an incorrect check from Field::get(). [#16617](https://github.com/ClickHouse/ClickHouse/pull/16617) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Apply SETTINGS clause as early as possible. It allows to modify more settings in the query. This closes [#3178](https://github.com/ClickHouse/ClickHouse/issues/3178). [#16619](https://github.com/ClickHouse/ClickHouse/pull/16619) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better update of ZooKeeper configuration in runtime. [#16630](https://github.com/ClickHouse/ClickHouse/pull/16630) ([sundyli](https://github.com/sundy-li)). +* Make the behaviour of `minMap` and `maxMap` more desireable. It will not skip zero values in the result. Fixes [#16087](https://github.com/ClickHouse/ClickHouse/issues/16087). [#16631](https://github.com/ClickHouse/ClickHouse/pull/16631) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Better diagnostics on parse errors in input data. Provide row number on `Cannot read all data` errors. [#16644](https://github.com/ClickHouse/ClickHouse/pull/16644) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Workaround for use S3 with nginx server as proxy. Nginx currenty does not accept urls with empty path like http://domain.com?delete, but vanilla aws-sdk-cpp produces this kind of urls. This commit uses patched aws-sdk-cpp version, which makes urls with "/" as path in this cases, like http://domain.com/?delete. [#16814](https://github.com/ClickHouse/ClickHouse/pull/16814) ([ianton-ru](https://github.com/ianton-ru)). + +#### Bug Fix +* Update jemalloc to fix percpu_arena with affinity mask. [#15035](https://github.com/ClickHouse/ClickHouse/pull/15035) ([Azat Khuzhin](https://github.com/azat)). +* Decrement the `ReadonlyReplica` metric when detaching read-only tables. This fixes [#15598](https://github.com/ClickHouse/ClickHouse/issues/15598). [#15592](https://github.com/ClickHouse/ClickHouse/pull/15592) ([sundyli](https://github.com/sundy-li)). +* Fixed bug with globs in S3 table function, region from URL was not applied to S3 client configuration. [#15646](https://github.com/ClickHouse/ClickHouse/pull/15646) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix error `Cannot add simple transform to empty Pipe` which happened while reading from `Buffer` table which has different structure than destination table. It was possible if destination table returned empty result for query. Fixes [#15529](https://github.com/ClickHouse/ClickHouse/issues/15529). [#15662](https://github.com/ClickHouse/ClickHouse/pull/15662) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Possibility to move part to another disk/volume if the first attempt was failed. [#15723](https://github.com/ClickHouse/ClickHouse/pull/15723) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix drop of materialized view with inner table in Atomic database (hangs all subsequent DROP TABLE due to hang of the worker thread, due to recursive DROP TABLE for inner table of MV). [#15743](https://github.com/ClickHouse/ClickHouse/pull/15743) ([Azat Khuzhin](https://github.com/azat)). +* Fix `select count()` inaccuracy for MaterializeMySQL. [#15767](https://github.com/ClickHouse/ClickHouse/pull/15767) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix exception `Block structure mismatch` in `SELECT ... ORDER BY DESC` queries which were executed after `ALTER MODIFY COLUMN` query. Fixes [#15800](https://github.com/ClickHouse/ClickHouse/issues/15800). [#15852](https://github.com/ClickHouse/ClickHouse/pull/15852) ([alesapin](https://github.com/alesapin)). +* Now exception will be thrown when `ALTER MODIFY COLUMN ... DEFAULT ...` has incompatible default with column type. Fixes [#15854](https://github.com/ClickHouse/ClickHouse/issues/15854). [#15858](https://github.com/ClickHouse/ClickHouse/pull/15858) ([alesapin](https://github.com/alesapin)). +* Fix possible deadlocks in RBAC. [#15875](https://github.com/ClickHouse/ClickHouse/pull/15875) ([Vitaly Baranov](https://github.com/vitlibar)). +* fixes [#12513](https://github.com/ClickHouse/ClickHouse/issues/12513) fix difference expressions with same alias when analyze queries again. [#15886](https://github.com/ClickHouse/ClickHouse/pull/15886) ([Winter Zhang](https://github.com/zhang2014)). +* Fix incorrect empty result for query from `Distributed` table if query has `WHERE`, `PREWHERE` and `GLOBAL IN`. Fixes [#15792](https://github.com/ClickHouse/ClickHouse/issues/15792). [#15933](https://github.com/ClickHouse/ClickHouse/pull/15933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed `DROP TABLE IF EXISTS` failure with `Table ... doesn't exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) Fixed `DROP/DETACH DATABASE` failure with `Table ... doesn't exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix a crash when database creation fails. [#15954](https://github.com/ClickHouse/ClickHouse/pull/15954) ([Winter Zhang](https://github.com/zhang2014)). +* Fix ambiguity in parsing of settings profiles: `CREATE USER ... SETTINGS profile readonly` is now considered as using a profile named `readonly`, not a setting named `profile` with the readonly constraint. This fixes [#15628](https://github.com/ClickHouse/ClickHouse/issues/15628). [#15982](https://github.com/ClickHouse/ClickHouse/pull/15982) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix rare segfaults when inserting into or selecting from MaterializedView and concurrently dropping target table (for Atomic database engine). [#15984](https://github.com/ClickHouse/ClickHouse/pull/15984) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Prevent replica hang for 5-10 mins when replication error happens after a period of inactivity. [#15987](https://github.com/ClickHouse/ClickHouse/pull/15987) ([filimonov](https://github.com/filimonov)). +* Allow to use direct layout for dictionaries with complex keys. [#16007](https://github.com/ClickHouse/ClickHouse/pull/16007) ([Anton Popov](https://github.com/CurtizJ)). +* Fix collate name & charset name parser and support `length = 0` for string type. [#16008](https://github.com/ClickHouse/ClickHouse/pull/16008) ([Winter Zhang](https://github.com/zhang2014)). +* Fix `ALTER MODIFY ... ORDER BY` query hang for `ReplicatedVersionedCollapsingMergeTree`. This fixes [#15980](https://github.com/ClickHouse/ClickHouse/issues/15980). [#16011](https://github.com/ClickHouse/ClickHouse/pull/16011) ([alesapin](https://github.com/alesapin)). +* Fix bug with MySQL database. When MySQL server used as database engine is down some queries raise Exception, because they try to get tables from disabled server, while it's unnecessary. For example, query `SELECT ... FROM system.parts` should work only with MergeTree tables and don't touch MySQL database at all. [#16032](https://github.com/ClickHouse/ClickHouse/pull/16032) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixes [#15780](https://github.com/ClickHouse/ClickHouse/issues/15780) regression, e.g. indexOf([1, 2, 3], toLowCardinality(1)) now is prohibited but it should not be. [#16038](https://github.com/ClickHouse/ClickHouse/pull/16038) ([Mike Kot](https://github.com/myrrc)). +* Fix segfault in some cases of wrong aggregation in lambdas. [#16082](https://github.com/ClickHouse/ClickHouse/pull/16082) ([Anton Popov](https://github.com/CurtizJ)). +* Fix the `clickhouse-local` crash when trying to do `OPTIMIZE` command. Fixes [#16076](https://github.com/ClickHouse/ClickHouse/issues/16076). [#16192](https://github.com/ClickHouse/ClickHouse/pull/16192) ([filimonov](https://github.com/filimonov)). +* Fix dictGet in sharding_key (and similar places, i.e. when the function context is stored permanently). [#16205](https://github.com/ClickHouse/ClickHouse/pull/16205) ([Azat Khuzhin](https://github.com/azat)). +* Fix the case when memory can be overallocated regardless to the limit. This closes [#14560](https://github.com/ClickHouse/ClickHouse/issues/14560). [#16206](https://github.com/ClickHouse/ClickHouse/pull/16206) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a possible memory leak during `GROUP BY` with string keys, caused by an error in `TwoLevelStringHashTable` implementation. [#16264](https://github.com/ClickHouse/ClickHouse/pull/16264) ([Amos Bird](https://github.com/amosbird)). +* Fixed the inconsistent behaviour when a part of return data could be dropped because the set for its filtration wasn't created. [#16308](https://github.com/ClickHouse/ClickHouse/pull/16308) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix processing of very large entries in replication queue. Very large entries may appear in ALTER queries if table structure is extremely large (near 1 MB). This fixes [#16307](https://github.com/ClickHouse/ClickHouse/issues/16307). [#16332](https://github.com/ClickHouse/ClickHouse/pull/16332) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix async Distributed INSERT w/ prefer_localhost_replica=0 and internal_replication. [#16358](https://github.com/ClickHouse/ClickHouse/pull/16358) ([Azat Khuzhin](https://github.com/azat)). +* Fix group by with totals/rollup/cube modifers and min/max functions over group by keys. Fixes [#16393](https://github.com/ClickHouse/ClickHouse/issues/16393). [#16397](https://github.com/ClickHouse/ClickHouse/pull/16397) ([Anton Popov](https://github.com/CurtizJ)). +* Fix DROP TABLE for Distributed (racy with INSERT). [#16409](https://github.com/ClickHouse/ClickHouse/pull/16409) ([Azat Khuzhin](https://github.com/azat)). +* Fix double free in case of exception in function `dictGet`. It could have happened if dictionary was loaded with error. [#16429](https://github.com/ClickHouse/ClickHouse/pull/16429) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Specifically crafter argument of `round` function with `Decimal` was leading to integer division by zero. This fixes [#13338](https://github.com/ClickHouse/ClickHouse/issues/13338). [#16451](https://github.com/ClickHouse/ClickHouse/pull/16451) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix rapid growth of metadata when using MySQL Master -> MySQL Slave -> ClickHouse MaterializeMySQL Engine, and `slave_parallel_worker` enabled on MySQL Slave, by properly shrinking GTID sets. This fixes [#15951](https://github.com/ClickHouse/ClickHouse/issues/15951). [#16504](https://github.com/ClickHouse/ClickHouse/pull/16504) ([TCeason](https://github.com/TCeason)). +* Now when parsing AVRO from input the LowCardinality is removed from type. Fixes [#16188](https://github.com/ClickHouse/ClickHouse/issues/16188). [#16521](https://github.com/ClickHouse/ClickHouse/pull/16521) ([Mike Kot](https://github.com/myrrc)). +* Fix query_thread_log.query_duration_ms unit. [#16563](https://github.com/ClickHouse/ClickHouse/pull/16563) ([Azat Khuzhin](https://github.com/azat)). +* Calculation of `DEFAULT` expressions was involving possible name collisions (that was very unlikely to encounter). This fixes [#9359](https://github.com/ClickHouse/ClickHouse/issues/9359). [#16612](https://github.com/ClickHouse/ClickHouse/pull/16612) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#16750](https://github.com/ClickHouse/ClickHouse/issues/16750): Fixed [#16081](https://github.com/ClickHouse/ClickHouse/issues/16081). [#16613](https://github.com/ClickHouse/ClickHouse/pull/16613) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#16758](https://github.com/ClickHouse/ClickHouse/issues/16758): This will fix optimize_read_in_order/optimize_aggregation_in_order with max_threads>0 and expression in ORDER BY. [#16637](https://github.com/ClickHouse/ClickHouse/pull/16637) ([Azat Khuzhin](https://github.com/azat)). +* The setting `max_parallel_replicas` worked incorrectly if the queried table has no sampling. This fixes [#5733](https://github.com/ClickHouse/ClickHouse/issues/5733). [#16675](https://github.com/ClickHouse/ClickHouse/pull/16675) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#16739](https://github.com/ClickHouse/ClickHouse/issues/16739): Fix `IN` operator over several columns and tuples with enabled `transform_null_in` setting. Fixes [#15310](https://github.com/ClickHouse/ClickHouse/issues/15310). [#16722](https://github.com/ClickHouse/ClickHouse/pull/16722) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#16784](https://github.com/ClickHouse/ClickHouse/issues/16784): Mask password in data_path in the system.distribution_queue. [#16727](https://github.com/ClickHouse/ClickHouse/pull/16727) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16785](https://github.com/ClickHouse/ClickHouse/issues/16785): Not for changelog. [#16757](https://github.com/ClickHouse/ClickHouse/pull/16757) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### Build/Testing/Packaging Improvement +* Simplify Sys/V init script. [#14135](https://github.com/ClickHouse/ClickHouse/pull/14135) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix illegal code style `&vector[idx]` in libhdfs3. This fixes libcxx debug build. See also https://github.com/ClickHouse-Extras/libhdfs3/pull/8 . [#15815](https://github.com/ClickHouse/ClickHouse/pull/15815) ([Amos Bird](https://github.com/amosbird)). +* Check for `#pragma once` in headers. [#15818](https://github.com/ClickHouse/ClickHouse/pull/15818) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Check for executable bit on non-executable files. People often accidentially commit executable files from Windows. [#15843](https://github.com/ClickHouse/ClickHouse/pull/15843) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* - Updated RBAC syntax tests. - Updated `INSERT` and `SELECT` RBAC tests. - New RBAC tests for `ALTER` privileges by @ritaank - New RBAC tests for `MATERIALIZED`, `LIVE`, and standard `VIEWS`. - New RBAC tests for public system tables. - New RBAC tests for `SHOW TABLES`. [#16044](https://github.com/ClickHouse/ClickHouse/pull/16044) ([MyroTk](https://github.com/MyroTk)). +* Refuse to build with AppleClang because it's difficult to find out version correspondence. This closes [#16072](https://github.com/ClickHouse/ClickHouse/issues/16072). [#16074](https://github.com/ClickHouse/ClickHouse/pull/16074) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add scipy to fasttest docker. [#16093](https://github.com/ClickHouse/ClickHouse/pull/16093) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add MySQL BinLog file check tool for MaterializeMySQL database engine. [#16223](https://github.com/ClickHouse/ClickHouse/pull/16223) ([Winter Zhang](https://github.com/zhang2014)). +* Add flaky check for stateless tests. [#16238](https://github.com/ClickHouse/ClickHouse/pull/16238) ([alesapin](https://github.com/alesapin)). +* Converting test tests/queries/0_stateless/01446_json_strings_each_row to a shell script. [#16247](https://github.com/ClickHouse/ClickHouse/pull/16247) ([vzakaznikov](https://github.com/vzakaznikov)). +* - None. [#16249](https://github.com/ClickHouse/ClickHouse/pull/16249) ([Denis Glazachev](https://github.com/traceon)). +* Fixing fails in LDAP external user directory tests. [#16363](https://github.com/ClickHouse/ClickHouse/pull/16363) ([vzakaznikov](https://github.com/vzakaznikov)). +* During config removal wrong exit code was expected. [#16365](https://github.com/ClickHouse/ClickHouse/pull/16365) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fix LDAP tests by grabbing log size after container is stopped. [#16440](https://github.com/ClickHouse/ClickHouse/pull/16440) ([vzakaznikov](https://github.com/vzakaznikov)). +* Improve generation of build files for `ya.make` build system (Arcadia). [#16700](https://github.com/ClickHouse/ClickHouse/pull/16700) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Other +* Use only |name_parts| as primary name source and auto-generate full name. [#16149](https://github.com/ClickHouse/ClickHouse/pull/16149) ([Ivan](https://github.com/abyss7)). +* Rename struct NullSink from ReadHelpers to NullOutput, because class NullSink exists in Processors/NullSink.h. It's needed to prevent redefinition of 'NullSink' error. [#16520](https://github.com/ClickHouse/ClickHouse/pull/16520) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NO CL CATEGORY + +* Try to make MergeTreeWriteAheadLog forward compatible. [#16094](https://github.com/ClickHouse/ClickHouse/pull/16094) ([nvartolomei](https://github.com/nvartolomei)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Write structure of table functions to metadata"'. [#15961](https://github.com/ClickHouse/ClickHouse/pull/15961) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "scipy"'. [#16156](https://github.com/ClickHouse/ClickHouse/pull/16156) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* NO CL ENTRY: 'Bump markdown from 3.2.1 to 3.3.2 in /docs/tools'. [#16180](https://github.com/ClickHouse/ClickHouse/pull/16180) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Revert "Optionally upload clickhouse binary in fast test"'. [#16333](https://github.com/ClickHouse/ClickHouse/pull/16333) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'minor fix.'. [#16335](https://github.com/ClickHouse/ClickHouse/pull/16335) ([Xianda Ke](https://github.com/kexianda)). +* NO CL ENTRY: 'Bump tornado from 5.1.1 to 6.1 in /docs/tools'. [#16590](https://github.com/ClickHouse/ClickHouse/pull/16590) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump mkdocs-macros-plugin from 0.4.17 to 0.4.20 in /docs/tools'. [#16692](https://github.com/ClickHouse/ClickHouse/pull/16692) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). + diff --git a/docs/changelogs/v20.11.3.3-stable.md b/docs/changelogs/v20.11.3.3-stable.md new file mode 100644 index 00000000000..55126a08ec4 --- /dev/null +++ b/docs/changelogs/v20.11.3.3-stable.md @@ -0,0 +1,5 @@ +### ClickHouse release v20.11.3.3-stable FIXME as compared to v20.11.2.1-stable + +#### Bug Fix +* Backported in [#16891](https://github.com/ClickHouse/ClickHouse/issues/16891): Fix rare silent crashes when query profiler is on and ClickHouse is installed on OS with glibc version that has (supposedly) broken asynchronous unwind tables for some functions. This fixes [#15301](https://github.com/ClickHouse/ClickHouse/issues/15301). This fixes [#13098](https://github.com/ClickHouse/ClickHouse/issues/13098). [#16846](https://github.com/ClickHouse/ClickHouse/pull/16846) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.11.4.13-stable.md b/docs/changelogs/v20.11.4.13-stable.md new file mode 100644 index 00000000000..76b4b10867e --- /dev/null +++ b/docs/changelogs/v20.11.4.13-stable.md @@ -0,0 +1,20 @@ +### ClickHouse release v20.11.4.13-stable FIXME as compared to v20.11.3.3-stable + +#### Improvement +* Backported in [#17032](https://github.com/ClickHouse/ClickHouse/issues/17032): Make it possible to connect to `clickhouse-server` secure endpoint which requires SNI. This is possible when `clickhouse-server` is hosted behind TLS proxy. [#16938](https://github.com/ClickHouse/ClickHouse/pull/16938) ([filimonov](https://github.com/filimonov)). + +#### Bug Fix +* Backported in [#17074](https://github.com/ClickHouse/ClickHouse/issues/17074): fixes [#16574](https://github.com/ClickHouse/ClickHouse/issues/16574) fixes [#16231](https://github.com/ClickHouse/ClickHouse/issues/16231) fix remote query failure when using 'if' suffix aggregate function. [#16610](https://github.com/ClickHouse/ClickHouse/pull/16610) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#17024](https://github.com/ClickHouse/ClickHouse/issues/17024): Fix crash when using `any` without any arguments. This is for [#16803](https://github.com/ClickHouse/ClickHouse/issues/16803) . cc @azat. [#16826](https://github.com/ClickHouse/ClickHouse/pull/16826) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#16881](https://github.com/ClickHouse/ClickHouse/issues/16881): Abort multipart upload if no data was written to WriteBufferFromS3. [#16840](https://github.com/ClickHouse/ClickHouse/pull/16840) ([Pavel Kovalenko](https://github.com/Jokser)). +* Backported in [#16948](https://github.com/ClickHouse/ClickHouse/issues/16948): Prevent clickhouse server crashes when using TimeSeriesGroupSum. [#16865](https://github.com/ClickHouse/ClickHouse/pull/16865) ([filimonov](https://github.com/filimonov)). +* Backported in [#17076](https://github.com/ClickHouse/ClickHouse/issues/17076): Fix possible error `Illegal type of argument` for queries with `ORDER BY`. Fixes [#16580](https://github.com/ClickHouse/ClickHouse/issues/16580). [#16928](https://github.com/ClickHouse/ClickHouse/pull/16928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17010](https://github.com/ClickHouse/ClickHouse/issues/17010): Install script should always create subdirs in config folders. This is only relevant for Docker build with custom config. [#16936](https://github.com/ClickHouse/ClickHouse/pull/16936) ([filimonov](https://github.com/filimonov)). +* Backported in [#16966](https://github.com/ClickHouse/ClickHouse/issues/16966): Blame info was not calculated correctly in `clickhouse-git-import`. [#16959](https://github.com/ClickHouse/ClickHouse/pull/16959) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17014](https://github.com/ClickHouse/ClickHouse/issues/17014): Fix possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17039](https://github.com/ClickHouse/ClickHouse/issues/17039): Reresolve the IP of the `format_avro_schema_registry_url` in case of errors. [#16985](https://github.com/ClickHouse/ClickHouse/pull/16985) ([filimonov](https://github.com/filimonov)). +* Backported in [#17093](https://github.com/ClickHouse/ClickHouse/issues/17093): Fixed wrong result in big integers (128, 256 bit) when casting from double. [#16986](https://github.com/ClickHouse/ClickHouse/pull/16986) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#17127](https://github.com/ClickHouse/ClickHouse/issues/17127): Avoid unnecessary network errors for remote queries which may be cancelled while execution, like queries with `LIMIT`. [#17006](https://github.com/ClickHouse/ClickHouse/pull/17006) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#17132](https://github.com/ClickHouse/ClickHouse/issues/17132): Fixed crash on `CREATE TABLE ... AS some_table` query when `some_table` was created `AS table_function()` Fixes [#16944](https://github.com/ClickHouse/ClickHouse/issues/16944). [#17072](https://github.com/ClickHouse/ClickHouse/pull/17072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17170](https://github.com/ClickHouse/ClickHouse/issues/17170): Fix bug when `ON CLUSTER` queries may hang forever for non-leader ReplicatedMergeTreeTables. [#17089](https://github.com/ClickHouse/ClickHouse/pull/17089) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v20.11.5.18-stable.md b/docs/changelogs/v20.11.5.18-stable.md new file mode 100644 index 00000000000..88c798984ab --- /dev/null +++ b/docs/changelogs/v20.11.5.18-stable.md @@ -0,0 +1,33 @@ +### ClickHouse release v20.11.5.18-stable FIXME as compared to v20.11.4.13-stable + +#### Performance Improvement +* Backported in [#17592](https://github.com/ClickHouse/ClickHouse/issues/17592): Fix performance of reading from `Merge` tables over huge number of `MergeTree` tables. Fixes [#7748](https://github.com/ClickHouse/ClickHouse/issues/7748). [#16988](https://github.com/ClickHouse/ClickHouse/pull/16988) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix +* Backported in [#17629](https://github.com/ClickHouse/ClickHouse/issues/17629): Throw error when use ColumnTransformer replace non exist column. [#16183](https://github.com/ClickHouse/ClickHouse/pull/16183) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#17158](https://github.com/ClickHouse/ClickHouse/issues/17158): Fixed uncontrolled growth of TDigest. [#16680](https://github.com/ClickHouse/ClickHouse/pull/16680) ([hrissan](https://github.com/hrissan)). +* Backported in [#17313](https://github.com/ClickHouse/ClickHouse/issues/17313): Return number of affected rows for INSERT queries via MySQL protocol. Previously ClickHouse used to always return 0, it's fixed. Fixes [#16605](https://github.com/ClickHouse/ClickHouse/issues/16605). [#16715](https://github.com/ClickHouse/ClickHouse/pull/16715) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#17342](https://github.com/ClickHouse/ClickHouse/issues/17342): TODO. [#16866](https://github.com/ClickHouse/ClickHouse/pull/16866) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17588](https://github.com/ClickHouse/ClickHouse/issues/17588): Fix optimization of group by with enabled setting `optimize_aggregators_of_group_by_keys` and joins. Fixes [#12604](https://github.com/ClickHouse/ClickHouse/issues/12604). [#16951](https://github.com/ClickHouse/ClickHouse/pull/16951) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17595](https://github.com/ClickHouse/ClickHouse/issues/17595): Fix order by optimization with monotonous functions. Fixes [#16107](https://github.com/ClickHouse/ClickHouse/issues/16107). [#16956](https://github.com/ClickHouse/ClickHouse/pull/16956) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17430](https://github.com/ClickHouse/ClickHouse/issues/17430): Bug fix for funciton fuzzBits, related issue: [#16980](https://github.com/ClickHouse/ClickHouse/issues/16980). [#17051](https://github.com/ClickHouse/ClickHouse/pull/17051) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#17194](https://github.com/ClickHouse/ClickHouse/issues/17194): Fix ColumnConst comparison which leads to crash. This fixed [#17088](https://github.com/ClickHouse/ClickHouse/issues/17088) . [#17135](https://github.com/ClickHouse/ClickHouse/pull/17135) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17558](https://github.com/ClickHouse/ClickHouse/issues/17558): Fix possible wrong index analysis when the types of the index comparison are different. This fixes [#17122](https://github.com/ClickHouse/ClickHouse/issues/17122). [#17145](https://github.com/ClickHouse/ClickHouse/pull/17145) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17396](https://github.com/ClickHouse/ClickHouse/issues/17396): Fix [#15235](https://github.com/ClickHouse/ClickHouse/issues/15235). When clickhouse-copier handle non-partitioned table, throws segfault error. [#17248](https://github.com/ClickHouse/ClickHouse/pull/17248) ([Qi Chen](https://github.com/kaka11chen)). +* Backported in [#17408](https://github.com/ClickHouse/ClickHouse/issues/17408): Fix set index invalidation when there are const columns in the subquery. This fixes [#17246](https://github.com/ClickHouse/ClickHouse/issues/17246) . [#17249](https://github.com/ClickHouse/ClickHouse/pull/17249) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17490](https://github.com/ClickHouse/ClickHouse/issues/17490): Fix crash while reading from `JOIN` table with `LowCardinality` types. Fixes [#17228](https://github.com/ClickHouse/ClickHouse/issues/17228). [#17397](https://github.com/ClickHouse/ClickHouse/pull/17397) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17494](https://github.com/ClickHouse/ClickHouse/issues/17494): Fix duplicates after `DISTINCT` which were possible because of incorrect optimization. Fixes [#17294](https://github.com/ClickHouse/ClickHouse/issues/17294). [#17296](https://github.com/ClickHouse/ClickHouse/pull/17296) ([li chengxiang](https://github.com/chengxianglibra)). [#17439](https://github.com/ClickHouse/ClickHouse/pull/17439) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17521](https://github.com/ClickHouse/ClickHouse/issues/17521): Fix `ORDER BY` with enabled setting `optimize_redundant_functions_in_order_by`. [#17471](https://github.com/ClickHouse/ClickHouse/pull/17471) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17534](https://github.com/ClickHouse/ClickHouse/issues/17534): Fix bug when mark cache size was underestimated by clickhouse. It may happen when there are a lot of tiny files with marks. [#17496](https://github.com/ClickHouse/ClickHouse/pull/17496) ([alesapin](https://github.com/alesapin)). +* Backported in [#17628](https://github.com/ClickHouse/ClickHouse/issues/17628): Fix alter query hang when the corresponding mutation was killed on the different replica. Fixes [#16953](https://github.com/ClickHouse/ClickHouse/issues/16953). [#17499](https://github.com/ClickHouse/ClickHouse/pull/17499) ([alesapin](https://github.com/alesapin)). +* Backported in [#17610](https://github.com/ClickHouse/ClickHouse/issues/17610): When clickhouse-client is used in interactive mode with multiline queries, single line comment was erronously extended till the end of query. This fixes [#13654](https://github.com/ClickHouse/ClickHouse/issues/13654). [#17565](https://github.com/ClickHouse/ClickHouse/pull/17565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17684](https://github.com/ClickHouse/ClickHouse/issues/17684): Exception `fmt::v7::format_error` can be logged in background for MergeTree tables. This fixes [#17613](https://github.com/ClickHouse/ClickHouse/issues/17613). [#17615](https://github.com/ClickHouse/ClickHouse/pull/17615) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17696](https://github.com/ClickHouse/ClickHouse/issues/17696): In might be determined incorrectly if cluster is circular- (cross-) replicated or not when executing `ON CLUSTER` query due to race condition when `pool_size` > 1. It's fixed. [#17640](https://github.com/ClickHouse/ClickHouse/pull/17640) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix optimize_trivial_count_query with partition predicate (backport [#16767](https://github.com/ClickHouse/ClickHouse/issues/16767) to 20.11). [#17644](https://github.com/ClickHouse/ClickHouse/pull/17644) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#17728](https://github.com/ClickHouse/ClickHouse/issues/17728): Fixed `Function not implemented` error when executing `RENAME` query in `Atomic` database with ClickHouse running on Windows Subsystem for Linux. Fixes [#17661](https://github.com/ClickHouse/ClickHouse/issues/17661). [#17664](https://github.com/ClickHouse/ClickHouse/pull/17664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17785](https://github.com/ClickHouse/ClickHouse/issues/17785): Fixed problem when ClickHouse fails to resume connection to MySQL servers. [#17681](https://github.com/ClickHouse/ClickHouse/pull/17681) ([Alexander Kazakov](https://github.com/Akazz)). +* Backported in [#17815](https://github.com/ClickHouse/ClickHouse/issues/17815): Do not restore parts from WAL if `in_memory_parts_enable_wal` is disabled. [#17802](https://github.com/ClickHouse/ClickHouse/pull/17802) ([detailyang](https://github.com/detailyang)). + +#### Build/Testing/Packaging Improvement +* Backported in [#17290](https://github.com/ClickHouse/ClickHouse/issues/17290): Update embedded timezone data to version 2020d (also update cctz to the latest master). [#17204](https://github.com/ClickHouse/ClickHouse/pull/17204) ([filimonov](https://github.com/filimonov)). + diff --git a/docs/changelogs/v20.11.6.6-stable.md b/docs/changelogs/v20.11.6.6-stable.md new file mode 100644 index 00000000000..f6c3e5fb8d6 --- /dev/null +++ b/docs/changelogs/v20.11.6.6-stable.md @@ -0,0 +1,14 @@ +### ClickHouse release v20.11.6.6-stable FIXME as compared to v20.11.5.18-stable + +#### Bug Fix +* Backported in [#17797](https://github.com/ClickHouse/ClickHouse/issues/17797): - Fix optimize_distributed_group_by_sharding_key for query with OFFSET only. [#16996](https://github.com/ClickHouse/ClickHouse/pull/16996) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#18394](https://github.com/ClickHouse/ClickHouse/issues/18394): Fix empty `system.stack_trace` table when server is running in daemon mode. [#17630](https://github.com/ClickHouse/ClickHouse/pull/17630) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#18044](https://github.com/ClickHouse/ClickHouse/issues/18044): Fix possible segfault in `topK` aggregate function. This closes [#17404](https://github.com/ClickHouse/ClickHouse/issues/17404). [#17845](https://github.com/ClickHouse/ClickHouse/pull/17845) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#18022](https://github.com/ClickHouse/ClickHouse/issues/18022): Trivial query optimization was producing wrong result if query contains ARRAY JOIN (so query is actually non trivial). [#17887](https://github.com/ClickHouse/ClickHouse/pull/17887) ([sundyli](https://github.com/sundy-li)). +* Backported in [#17979](https://github.com/ClickHouse/ClickHouse/issues/17979): fixes [#15187](https://github.com/ClickHouse/ClickHouse/issues/15187) fixes [#17912](https://github.com/ClickHouse/ClickHouse/issues/17912) support convert MySQL prefix index for MaterializeMySQL CC: @tavplubix. [#17944](https://github.com/ClickHouse/ClickHouse/pull/17944) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#18080](https://github.com/ClickHouse/ClickHouse/issues/18080): Fixed `std::out_of_range: basic_string` in S3 URL parsing. [#18059](https://github.com/ClickHouse/ClickHouse/pull/18059) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#18180](https://github.com/ClickHouse/ClickHouse/issues/18180): Fix `Unknown setting profile` error on attempt to set settings profile. [#18167](https://github.com/ClickHouse/ClickHouse/pull/18167) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#18358](https://github.com/ClickHouse/ClickHouse/issues/18358): fixes [#18186](https://github.com/ClickHouse/ClickHouse/issues/18186) fixes [#16372](https://github.com/ClickHouse/ClickHouse/issues/16372) fix unique key convert crash in MaterializeMySQL database engine. [#18211](https://github.com/ClickHouse/ClickHouse/pull/18211) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#18259](https://github.com/ClickHouse/ClickHouse/issues/18259): Fix key comparison between Enum and Int types. This fixes [#17989](https://github.com/ClickHouse/ClickHouse/issues/17989). [#18214](https://github.com/ClickHouse/ClickHouse/pull/18214) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#18297](https://github.com/ClickHouse/ClickHouse/issues/18297): - Fixed issue when `clickhouse-odbc-bridge` process is unreachable by server on machines with dual IPv4/IPv6 stack; - Fixed issue when ODBC dictionary updates are performed using malformed queries and/or cause crashes; Possibly closes [#14489](https://github.com/ClickHouse/ClickHouse/issues/14489). [#18278](https://github.com/ClickHouse/ClickHouse/pull/18278) ([Denis Glazachev](https://github.com/traceon)). + diff --git a/docs/changelogs/v20.11.7.16-stable.md b/docs/changelogs/v20.11.7.16-stable.md new file mode 100644 index 00000000000..a4160b47556 --- /dev/null +++ b/docs/changelogs/v20.11.7.16-stable.md @@ -0,0 +1,62 @@ +### ClickHouse release v20.11.7.16-stable FIXME as compared to v20.11.6.6-stable + +#### Improvement +* Backported in [#19147](https://github.com/ClickHouse/ClickHouse/issues/19147): Explicitly set uid / gid of clickhouse user & group to the fixed values (101) in clickhouse-server images. [#19096](https://github.com/ClickHouse/ClickHouse/pull/19096) ([filimonov](https://github.com/filimonov)). + +#### Bug Fix +* Backported in [#18268](https://github.com/ClickHouse/ClickHouse/issues/18268): Fix indeterministic functions with predicate optimizer. This fixes [#17244](https://github.com/ClickHouse/ClickHouse/issues/17244). [#17273](https://github.com/ClickHouse/ClickHouse/pull/17273) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#19657](https://github.com/ClickHouse/ClickHouse/issues/19657): fix data type convert issue for mysql engine ... [#18124](https://github.com/ClickHouse/ClickHouse/pull/18124) ([bo zeng](https://github.com/mis98zb)). +* Backported in [#18165](https://github.com/ClickHouse/ClickHouse/issues/18165): Fix error when query `MODIFY COLUMN ... REMOVE TTL` doesn't actually remove column TTL. [#18130](https://github.com/ClickHouse/ClickHouse/pull/18130) ([alesapin](https://github.com/alesapin)). +* Backported in [#19056](https://github.com/ClickHouse/ClickHouse/issues/19056): Fix inserting a row with default value in case of parsing error in the last column. Fixes [#17712](https://github.com/ClickHouse/ClickHouse/issues/17712). [#18182](https://github.com/ClickHouse/ClickHouse/pull/18182) ([Jianmei Zhang](https://github.com/zhangjmruc)). +* Backported in [#18230](https://github.com/ClickHouse/ClickHouse/issues/18230): Fix possible incomplete query result while reading from `MergeTree*` in case of read backoff (message ` MergeTreeReadPool: Will lower number of threads` in logs). Was introduced in [#16423](https://github.com/ClickHouse/ClickHouse/issues/16423). Fixes [#18137](https://github.com/ClickHouse/ClickHouse/issues/18137). [#18216](https://github.com/ClickHouse/ClickHouse/pull/18216) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#18632](https://github.com/ClickHouse/ClickHouse/issues/18632): `SELECT JOIN` now requires the `SELECT` privilege on each of the joined tables. This PR fixes [#17654](https://github.com/ClickHouse/ClickHouse/issues/17654). [#18232](https://github.com/ClickHouse/ClickHouse/pull/18232) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#18426](https://github.com/ClickHouse/ClickHouse/issues/18426): Fix possible crashes in aggregate functions with combinator `Distinct`, while using two-level aggregation. Fixes [#17682](https://github.com/ClickHouse/ClickHouse/issues/17682). [#18365](https://github.com/ClickHouse/ClickHouse/pull/18365) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#19160](https://github.com/ClickHouse/ClickHouse/issues/19160): Fix index analysis of binary functions with constant argument which leads to wrong query results. This fixes [#18364](https://github.com/ClickHouse/ClickHouse/issues/18364). [#18373](https://github.com/ClickHouse/ClickHouse/pull/18373) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#18430](https://github.com/ClickHouse/ClickHouse/issues/18430): Fix filling table `system.settings_profile_elements`. This PR fixes [#18231](https://github.com/ClickHouse/ClickHouse/issues/18231). [#18379](https://github.com/ClickHouse/ClickHouse/pull/18379) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#18483](https://github.com/ClickHouse/ClickHouse/issues/18483): Restrict merges from wide to compact parts. In case of vertical merge it led to broken result part. [#18381](https://github.com/ClickHouse/ClickHouse/pull/18381) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#18472](https://github.com/ClickHouse/ClickHouse/issues/18472): Fixed `value is too short` error when executing `toType(...)` functions (`toDate`, `toUInt32`, etc) with argument of type `Nullable(String)`. Now such functions return `NULL` on parsing errors instead of throwing exception. Fixes [#7673](https://github.com/ClickHouse/ClickHouse/issues/7673). [#18445](https://github.com/ClickHouse/ClickHouse/pull/18445) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19718](https://github.com/ClickHouse/ClickHouse/issues/19718): Disable constant folding for subqueries on the analysis stage, when the result cannot be calculated. [#18446](https://github.com/ClickHouse/ClickHouse/pull/18446) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#18532](https://github.com/ClickHouse/ClickHouse/issues/18532): Proper support for 12AM in `parseDateTimeBestEffort` function. This fixes [#18402](https://github.com/ClickHouse/ClickHouse/issues/18402). [#18449](https://github.com/ClickHouse/ClickHouse/pull/18449) ([vladimir-golovchenko](https://github.com/vladimir-golovchenko)). +* Backported in [#18503](https://github.com/ClickHouse/ClickHouse/issues/18503): Disable write with AIO during merges because it can lead to extremely rare data corruption of primary key columns during merge. [#18481](https://github.com/ClickHouse/ClickHouse/pull/18481) ([alesapin](https://github.com/alesapin)). +* Backported in [#18600](https://github.com/ClickHouse/ClickHouse/issues/18600): Fix bug which may lead to `ALTER` queries hung after corresponding mutation kill. Found by thread fuzzer. [#18518](https://github.com/ClickHouse/ClickHouse/pull/18518) ([alesapin](https://github.com/alesapin)). +* Backported in [#18576](https://github.com/ClickHouse/ClickHouse/issues/18576): Fix possible `Pipeline stuck` error while using `ORDER BY` after subquery with `RIGHT` or `FULL` join. [#18550](https://github.com/ClickHouse/ClickHouse/pull/18550) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#18607](https://github.com/ClickHouse/ClickHouse/issues/18607): Add FixedString Data type support. I'll get this exception "Code: 50, e.displayText() = DB::Exception: Unsupported type FixedString(1)" when replicating data from MySQL to ClickHouse. This patch fixes bug [#18450](https://github.com/ClickHouse/ClickHouse/issues/18450) Also fixes [#6556](https://github.com/ClickHouse/ClickHouse/issues/6556). [#18553](https://github.com/ClickHouse/ClickHouse/pull/18553) ([awesomeleo](https://github.com/awesomeleo)). +* Backported in [#18735](https://github.com/ClickHouse/ClickHouse/issues/18735): Fix Logger with unmatched arg size. [#18717](https://github.com/ClickHouse/ClickHouse/pull/18717) ([sundyli](https://github.com/sundy-li)). +* Backported in [#18947](https://github.com/ClickHouse/ClickHouse/issues/18947): Fixed `Attempt to read after eof` error when trying to `CAST` `NULL` from `Nullable(String)` to `Nullable(Decimal(P, S))`. Now function `CAST` returns `NULL` when it cannot parse decimal from nullable string. Fixes [#7690](https://github.com/ClickHouse/ClickHouse/issues/7690). [#18718](https://github.com/ClickHouse/ClickHouse/pull/18718) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#18801](https://github.com/ClickHouse/ClickHouse/issues/18801): Asynchronous distributed INSERTs can be rejected by the server if the setting `network_compression_method` is globally set to non-default value. This fixes [#18741](https://github.com/ClickHouse/ClickHouse/issues/18741). [#18776](https://github.com/ClickHouse/ClickHouse/pull/18776) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#18838](https://github.com/ClickHouse/ClickHouse/issues/18838): Fix *If combinator with unary function and Nullable types. [#18806](https://github.com/ClickHouse/ClickHouse/pull/18806) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#18908](https://github.com/ClickHouse/ClickHouse/issues/18908): Fix possible hang at shutdown in clickhouse-local. This fixes [#18891](https://github.com/ClickHouse/ClickHouse/issues/18891). [#18893](https://github.com/ClickHouse/ClickHouse/pull/18893) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19113](https://github.com/ClickHouse/ClickHouse/issues/19113): Attach partition should reset the mutation. [#18804](https://github.com/ClickHouse/ClickHouse/issues/18804). [#18935](https://github.com/ClickHouse/ClickHouse/pull/18935) ([fastio](https://github.com/fastio)). +* Backported in [#18997](https://github.com/ClickHouse/ClickHouse/issues/18997): Fix bug when mutation with some escaped text (like `ALTER ... UPDATE e = CAST('foo', 'Enum8(\'foo\' = 1')` serialized incorrectly. Fixes [#18878](https://github.com/ClickHouse/ClickHouse/issues/18878). [#18944](https://github.com/ClickHouse/ClickHouse/pull/18944) ([alesapin](https://github.com/alesapin)). +* Backported in [#19194](https://github.com/ClickHouse/ClickHouse/issues/19194): Fixed very rare deadlock at shutdown. [#18977](https://github.com/ClickHouse/ClickHouse/pull/18977) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19092](https://github.com/ClickHouse/ClickHouse/issues/19092): Disable `optimize_move_functions_out_of_any` because optimization is not always correct. This closes [#18051](https://github.com/ClickHouse/ClickHouse/issues/18051). This closes [#18973](https://github.com/ClickHouse/ClickHouse/issues/18973). [#18981](https://github.com/ClickHouse/ClickHouse/pull/18981) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19050](https://github.com/ClickHouse/ClickHouse/issues/19050): Fix inserting of `LowCardinality` column to table with `TinyLog` engine. Fixes [#18629](https://github.com/ClickHouse/ClickHouse/issues/18629). [#19010](https://github.com/ClickHouse/ClickHouse/pull/19010) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19082](https://github.com/ClickHouse/ClickHouse/issues/19082): Fix possible error `Expected single dictionary argument for function` if use function `ignore` with `LowCardinality` argument. Fixes [#14275](https://github.com/ClickHouse/ClickHouse/issues/14275). [#19016](https://github.com/ClickHouse/ClickHouse/pull/19016) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19049](https://github.com/ClickHouse/ClickHouse/issues/19049): Make sure `groupUniqArray` returns correct type for argument of Enum type. This closes [#17875](https://github.com/ClickHouse/ClickHouse/issues/17875). [#19019](https://github.com/ClickHouse/ClickHouse/pull/19019) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19125](https://github.com/ClickHouse/ClickHouse/issues/19125): Restrict `MODIFY TTL` queries for `MergeTree` tables created in old syntax. Previously the query succeeded, but actually it had no effect. [#19064](https://github.com/ClickHouse/ClickHouse/pull/19064) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#19564](https://github.com/ClickHouse/ClickHouse/issues/19564): Fixed `There is no checkpoint` error when inserting data through http interface using `Template` or `CustomSeparated` format. Fixes [#19021](https://github.com/ClickHouse/ClickHouse/issues/19021). [#19072](https://github.com/ClickHouse/ClickHouse/pull/19072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19231](https://github.com/ClickHouse/ClickHouse/issues/19231): Fix startup bug when clickhouse was not able to read compression codec from `LowCardinality(Nullable(...))` and throws exception `Attempt to read after EOF`. Fixes [#18340](https://github.com/ClickHouse/ClickHouse/issues/18340). [#19101](https://github.com/ClickHouse/ClickHouse/pull/19101) ([alesapin](https://github.com/alesapin)). +* Backported in [#19181](https://github.com/ClickHouse/ClickHouse/issues/19181): Fix infinite reading from file in `ORC` format (was introduced in [#10580](https://github.com/ClickHouse/ClickHouse/issues/10580)). Fixes [#19095](https://github.com/ClickHouse/ClickHouse/issues/19095). [#19134](https://github.com/ClickHouse/ClickHouse/pull/19134) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19268](https://github.com/ClickHouse/ClickHouse/issues/19268): Fix bug when concurrent `ALTER` and `DROP` queries may hang while processing ReplicatedMergeTree table. [#19237](https://github.com/ClickHouse/ClickHouse/pull/19237) ([alesapin](https://github.com/alesapin)). +* Backported in [#19665](https://github.com/ClickHouse/ClickHouse/issues/19665): Fix error `Cannot convert column now64() because it is constant but values of constants are different in source and result`. Continuation of [#7156](https://github.com/ClickHouse/ClickHouse/issues/7156). [#19316](https://github.com/ClickHouse/ClickHouse/pull/19316) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19555](https://github.com/ClickHouse/ClickHouse/issues/19555): Fix system.parts _state column (LOGICAL_ERROR when querying this column, due to incorrect order). [#19346](https://github.com/ClickHouse/ClickHouse/pull/19346) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19470](https://github.com/ClickHouse/ClickHouse/issues/19470): - Fix default value in join types with non-zero default (e.g. some Enums). Closes [#18197](https://github.com/ClickHouse/ClickHouse/issues/18197). [#19360](https://github.com/ClickHouse/ClickHouse/pull/19360) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#19439](https://github.com/ClickHouse/ClickHouse/issues/19439): Fix possible buffer overflow in Uber H3 library. See https://github.com/uber/h3/issues/392. This closes [#19219](https://github.com/ClickHouse/ClickHouse/issues/19219). [#19383](https://github.com/ClickHouse/ClickHouse/pull/19383) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19614](https://github.com/ClickHouse/ClickHouse/issues/19614): Fixed very rare bug that might cause mutation to hang after `DROP/DETACH/REPLACE/MOVE PARTITION`. It was partially fixed by [#15537](https://github.com/ClickHouse/ClickHouse/issues/15537) for the most cases. [#19443](https://github.com/ClickHouse/ClickHouse/pull/19443) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19669](https://github.com/ClickHouse/ClickHouse/issues/19669): Mark distributed batch as broken in case of empty data block in one of files. [#19449](https://github.com/ClickHouse/ClickHouse/pull/19449) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19509](https://github.com/ClickHouse/ClickHouse/issues/19509): Buffer overflow (on memory read) was possible if `addMonth` function was called with specifically crafted arguments. This fixes [#19441](https://github.com/ClickHouse/ClickHouse/issues/19441). This fixes [#19413](https://github.com/ClickHouse/ClickHouse/issues/19413). [#19472](https://github.com/ClickHouse/ClickHouse/pull/19472) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19538](https://github.com/ClickHouse/ClickHouse/issues/19538): Fix SIGSEGV with merge_tree_min_rows_for_concurrent_read/merge_tree_min_bytes_for_concurrent_read=0/UINT64_MAX. [#19528](https://github.com/ClickHouse/ClickHouse/pull/19528) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19641](https://github.com/ClickHouse/ClickHouse/issues/19641): Query CREATE DICTIONARY id expression fix. [#19571](https://github.com/ClickHouse/ClickHouse/pull/19571) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#19637](https://github.com/ClickHouse/ClickHouse/issues/19637): `DROP/DETACH TABLE table ON CLUSTER cluster SYNC` query might hang, it's fixed. Fixes [#19568](https://github.com/ClickHouse/ClickHouse/issues/19568). [#19572](https://github.com/ClickHouse/ClickHouse/pull/19572) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19653](https://github.com/ClickHouse/ClickHouse/issues/19653): Fix use-after-free of the CompressedWriteBuffer in Connection after disconnect. [#19599](https://github.com/ClickHouse/ClickHouse/pull/19599) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19740](https://github.com/ClickHouse/ClickHouse/issues/19740): Fix wrong result of function `neighbor` for `LowCardinality` argument. Fixes [#10333](https://github.com/ClickHouse/ClickHouse/issues/10333). [#19617](https://github.com/ClickHouse/ClickHouse/pull/19617) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19698](https://github.com/ClickHouse/ClickHouse/issues/19698): Some functions with big integers may cause segfault. Big integers is experimental feature. This closes [#19667](https://github.com/ClickHouse/ClickHouse/issues/19667). [#19672](https://github.com/ClickHouse/ClickHouse/pull/19672) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19817](https://github.com/ClickHouse/ClickHouse/issues/19817): Fix a segmentation fault in `bitmapAndnot` function. Fixes [#19668](https://github.com/ClickHouse/ClickHouse/issues/19668). [#19713](https://github.com/ClickHouse/ClickHouse/pull/19713) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#19880](https://github.com/ClickHouse/ClickHouse/issues/19880): Fixed stack overflow when using accurate comparison of arithmetic type with string type. [#19773](https://github.com/ClickHouse/ClickHouse/pull/19773) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19812](https://github.com/ClickHouse/ClickHouse/issues/19812): In previous versions, unusual arguments for function arrayEnumerateUniq may cause crash or infinite loop. This closes [#19787](https://github.com/ClickHouse/ClickHouse/issues/19787). [#19788](https://github.com/ClickHouse/ClickHouse/pull/19788) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19940](https://github.com/ClickHouse/ClickHouse/issues/19940): Deadlock was possible if system.text_log is enabled. This fixes [#19874](https://github.com/ClickHouse/ClickHouse/issues/19874). [#19875](https://github.com/ClickHouse/ClickHouse/pull/19875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19936](https://github.com/ClickHouse/ClickHouse/issues/19936): BloomFilter index crash fix. Fixes [#19757](https://github.com/ClickHouse/ClickHouse/issues/19757). [#19884](https://github.com/ClickHouse/ClickHouse/pull/19884) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Build/Testing/Packaging Improvement +* Backported in [#18543](https://github.com/ClickHouse/ClickHouse/issues/18543): Update timezones info to 2020e. [#18531](https://github.com/ClickHouse/ClickHouse/pull/18531) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v20.12.1.5236-prestable.md b/docs/changelogs/v20.12.1.5236-prestable.md new file mode 100644 index 00000000000..f4ecc83e176 --- /dev/null +++ b/docs/changelogs/v20.12.1.5236-prestable.md @@ -0,0 +1,98 @@ +### ClickHouse release v20.12.1.5236-prestable FIXME as compared to v20.11.1.5109-prestable + +#### Backward Incompatible Change +* Accept user settings related to file formats (e.g. `format_csv_delimiter`) in the `SETTINGS` clause when creating a table that uses `File` engine, and use these settings in all `INSERT`s and `SELECT`s. The file format settings changed in the current user session, or in the `SETTINGS` clause of a DML query itself, no longer affect the query. [#16591](https://github.com/ClickHouse/ClickHouse/pull/16591) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Enable `use_compact_format_in_distributed_parts_names` by default (see the documentation for the reference). [#16728](https://github.com/ClickHouse/ClickHouse/pull/16728) ([Azat Khuzhin](https://github.com/azat)). + +#### New Feature +* Added new ALTER UPDATE/DELETE IN PARTITION syntax. [#13403](https://github.com/ClickHouse/ClickHouse/pull/13403) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Add StorageEmbeddedRocksdb Engine. [#15073](https://github.com/ClickHouse/ClickHouse/pull/15073) ([sundyli](https://github.com/sundy-li)). +* Introduce the query `ALTER TABLE ... DROP|DETACH PART 'part_name'`. [#15511](https://github.com/ClickHouse/ClickHouse/pull/15511) ([nvartolomei](https://github.com/nvartolomei)). +* Make it possible to change the path to history file in `clickhouse-client` using the `--history_file` parameter. [#15960](https://github.com/ClickHouse/ClickHouse/pull/15960) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated DateTime, DateTime64 formatting to accept string Date literal format. [#16040](https://github.com/ClickHouse/ClickHouse/pull/16040) ([Maksim Kita](https://github.com/kitaisreal)). +* Add setting `aggregate_functions_null_for_empty`, this option will rewrite all aggregate functions in a query, adding -OrNull suffix to them. fix [10273](https://github.com/ClickHouse/ClickHouse/issues/10273). [#16123](https://github.com/ClickHouse/ClickHouse/pull/16123) ([flynn](https://github.com/ucasfl)). +* Add COLLATE support for Nullable, LowCardinality, Array and Tuple, where nested type is String. Also refactor the code associated with collations in ColumnString.cpp. [#16273](https://github.com/ClickHouse/ClickHouse/pull/16273) ([Kruglov Pavel](https://github.com/Avogar)). +* Possibility to distribute the merges between different replicas. Introduces the `execute_merges_on_single_replica_time_threshold` mergetree setting. [#16424](https://github.com/ClickHouse/ClickHouse/pull/16424) ([filimonov](https://github.com/filimonov)). +* add `*.xz` compression/decompression support.It enables using `*.xz` in `file()` function.This closes [#8828](https://github.com/ClickHouse/ClickHouse/issues/8828). [#16578](https://github.com/ClickHouse/ClickHouse/pull/16578) ([Abi Palagashvili](https://github.com/fibersel)). +* Add new `cmath` functions: - acosh - asinh - atan2 - atanh - cosh - hypot - log1p - sinh. [#16636](https://github.com/ClickHouse/ClickHouse/pull/16636) ([Konstantin Malanchev](https://github.com/hombit)). +* Add a possibility to input enum value as it's id in TSV and CSV formats by default. [#16834](https://github.com/ClickHouse/ClickHouse/pull/16834) ([Kruglov Pavel](https://github.com/Avogar)). +* New tcpPort() function returns TCP port listened by this server. [#17134](https://github.com/ClickHouse/ClickHouse/pull/17134) ([Ivan](https://github.com/abyss7)). + +#### Performance Improvement +* Now we can safely prune partitions with exact match. Useful case: Suppose table is partitioned by intHash64(x) % 100 and the query has condition on intHash64(x) % 100 verbatim, not on x. [#16253](https://github.com/ClickHouse/ClickHouse/pull/16253) ([Amos Bird](https://github.com/amosbird)). +* Use Floyd-Rivest algorithm, it should be the best for the ClickHouse use case of partial sorting. Bechmarks are in https://github.com/danlark1/miniselect and [here](https://drive.google.com/drive/folders/1DHEaeXgZuX6AJ9eByeZ8iQVQv0ueP8XM). [#16825](https://github.com/ClickHouse/ClickHouse/pull/16825) ([Daniel Kutenin](https://github.com/danlark1)). + +#### Improvement +* Add `VIEW` subquery description to `EXPLAIN`. Limit push down optimisation for `VIEW`. Add local replicas of `Distributed` to query plan. [#14936](https://github.com/ClickHouse/ClickHouse/pull/14936) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Made `indexOf()` use BloomFilter. [#14977](https://github.com/ClickHouse/ClickHouse/pull/14977) ([achimbab](https://github.com/achimbab)). +* Throw exception about right sync privileges when MySQL sync user has error privileges. [#15977](https://github.com/ClickHouse/ClickHouse/pull/15977) ([TCeason](https://github.com/TCeason)). +* Fix possible stack overflow if a loop of materialized views is created. This closes [#15732](https://github.com/ClickHouse/ClickHouse/issues/15732). [#16048](https://github.com/ClickHouse/ClickHouse/pull/16048) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support SNI in https connections to remote resources. This will allow to connect to Cloudflare servers that require SNI. This fixes [#10055](https://github.com/ClickHouse/ClickHouse/issues/10055). [#16252](https://github.com/ClickHouse/ClickHouse/pull/16252) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now `ReplicatedMergeTree` tree engines family uses a separate thread pool for replicated fetches. Size of the pool limited by setting `background_fetches_pool_size` which can be tuned with a server restart. The default value of the setting is 3 and it means that the maximum amount of parallel fetches is equal to 3 (and it allows to utilize 10G network). Fixes #520. [#16390](https://github.com/ClickHouse/ClickHouse/pull/16390) ([alesapin](https://github.com/alesapin)). +* Now, `` configuration can be changed in `config.xml` and reloaded without server startup. [#16627](https://github.com/ClickHouse/ClickHouse/pull/16627) ([Amos Bird](https://github.com/amosbird)). +* Allow reinterpret between integers and floats of the same size. fix [16640](https://github.com/ClickHouse/ClickHouse/issues/16640). [#16657](https://github.com/ClickHouse/ClickHouse/pull/16657) ([flynn](https://github.com/ucasfl)). +* Workaround for use S3 with nginx server as proxy. Nginx currenty does not accept urls with empty path like `http://domain.com?delete`, but vanilla aws-sdk-cpp produces this kind of urls. This commit uses patched aws-sdk-cpp version, which makes urls with "/" as path in this cases, like `http://domain.com/?delete`. [#16709](https://github.com/ClickHouse/ClickHouse/pull/16709) ([ianton-ru](https://github.com/ianton-ru)). +* Remove empty directories for async INSERT at start of Distributed engine. [#16729](https://github.com/ClickHouse/ClickHouse/pull/16729) ([Azat Khuzhin](https://github.com/azat)). +* Usability improvement: better suggestions in syntax error message when `CODEC` expression is misplaced in `CREATE TABLE` query. This fixes [#12493](https://github.com/ClickHouse/ClickHouse/issues/12493). [#16768](https://github.com/ClickHouse/ClickHouse/pull/16768) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better exception message when configuration for distributed DDL is absent. This fixes [#5075](https://github.com/ClickHouse/ClickHouse/issues/5075). [#16769](https://github.com/ClickHouse/ClickHouse/pull/16769) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Apply `use_compact_format_in_distributed_parts_names` for each INSERT (with internal_replication). [#16788](https://github.com/ClickHouse/ClickHouse/pull/16788) ([Azat Khuzhin](https://github.com/azat)). +* Server refused to startup with exception message if wrong config is given (`metric_log`.`collect_interval_milliseconds` is missing). [#16815](https://github.com/ClickHouse/ClickHouse/pull/16815) ([Ivan](https://github.com/abyss7)). +* Add cutToFirstSignificantSubdomainWithWWW(). [#16845](https://github.com/ClickHouse/ClickHouse/pull/16845) ([Azat Khuzhin](https://github.com/azat)). +* Throw an informative error message when doing ATTACH/DETACH TABLE . Before this PR, `detach table ` works but leads to an ill-formed in-memory metadata. [#16885](https://github.com/ClickHouse/ClickHouse/pull/16885) ([Amos Bird](https://github.com/amosbird)). +* Remove empty parts after they were pruned by TTL, mutation, or collapsing merge algorithm. [#16895](https://github.com/ClickHouse/ClickHouse/pull/16895) ([Anton Popov](https://github.com/CurtizJ)). +* Make it possible to connect to `clickhouse-server` secure endpoint which requires SNI. This is possible when `clickhouse-server` is hosted behind TLS proxy. [#16938](https://github.com/ClickHouse/ClickHouse/pull/16938) ([filimonov](https://github.com/filimonov)). +* Set default `host` and `port` parameters for `SOURCE(CLICKHOUSE(...))` to current instance and set default `user` value to `'default'`. [#16997](https://github.com/ClickHouse/ClickHouse/pull/16997) ([Vladimir C](https://github.com/vdimir)). +* Add ability to output all rows as a JSON array in the `JSONEachRow` format, controlled by the `output_format_json_array_of_rows` setting. [#17152](https://github.com/ClickHouse/ClickHouse/pull/17152) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow formatting named tuples as JSON objects when using JSON input/output formats, controlled by the `output_format_json_named_tuples_as_objects` setting, disabled by default. [#17175](https://github.com/ClickHouse/ClickHouse/pull/17175) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Correct grammar in error message in JSONEachRow, JSONCompactEachRow, and RegexpRow input formats. [#17205](https://github.com/ClickHouse/ClickHouse/pull/17205) ([nico piderman](https://github.com/sneako)). + +#### Bug Fix +* fixes [#16574](https://github.com/ClickHouse/ClickHouse/issues/16574) fixes [#16231](https://github.com/ClickHouse/ClickHouse/issues/16231) fix remote query failure when using 'if' suffix aggregate function. [#16610](https://github.com/ClickHouse/ClickHouse/pull/16610) ([Winter Zhang](https://github.com/zhang2014)). +* Fixed [#16081](https://github.com/ClickHouse/ClickHouse/issues/16081). [#16613](https://github.com/ClickHouse/ClickHouse/pull/16613) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* This will fix optimize_read_in_order/optimize_aggregation_in_order with max_threads>0 and expression in ORDER BY. [#16637](https://github.com/ClickHouse/ClickHouse/pull/16637) ([Azat Khuzhin](https://github.com/azat)). +* Fixed uncontrolled growth of TDigest. [#16680](https://github.com/ClickHouse/ClickHouse/pull/16680) ([hrissan](https://github.com/hrissan)). +* Turn off parallel parsing when there is no enough memory for all threads to work simultaneously. Also there could be exceptions like "Memory limit exceeded" when somebody will try to insert extremely huge rows (> min_chunk_bytes_for_parallel_parsing), because each piece to parse has to be independent set of strings (one or more). [#16721](https://github.com/ClickHouse/ClickHouse/pull/16721) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix `IN` operator over several columns and tuples with enabled `transform_null_in` setting. Fixes [#15310](https://github.com/ClickHouse/ClickHouse/issues/15310). [#16722](https://github.com/ClickHouse/ClickHouse/pull/16722) ([Anton Popov](https://github.com/CurtizJ)). +* Mask password in data_path in the system.distribution_queue. [#16727](https://github.com/ClickHouse/ClickHouse/pull/16727) ([Azat Khuzhin](https://github.com/azat)). +* Not for changelog. [#16757](https://github.com/ClickHouse/ClickHouse/pull/16757) ([Alexander Tokmakov](https://github.com/tavplubix)). +* If no memory can be allocated while writing table metadata on disk, broken metadata file can be written. [#16772](https://github.com/ClickHouse/ClickHouse/pull/16772) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix crash when using `any` without any arguments. This is for [#16803](https://github.com/ClickHouse/ClickHouse/issues/16803) . cc @azat. [#16826](https://github.com/ClickHouse/ClickHouse/pull/16826) ([Amos Bird](https://github.com/amosbird)). +* Abort multipart upload if no data was written to WriteBufferFromS3. [#16840](https://github.com/ClickHouse/ClickHouse/pull/16840) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix rare silent crashes when query profiler is on and ClickHouse is installed on OS with glibc version that has (supposedly) broken asynchronous unwind tables for some functions. This fixes [#15301](https://github.com/ClickHouse/ClickHouse/issues/15301). This fixes [#13098](https://github.com/ClickHouse/ClickHouse/issues/13098). [#16846](https://github.com/ClickHouse/ClickHouse/pull/16846) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Prevent clickhouse server crashes when using TimeSeriesGroupSum. [#16865](https://github.com/ClickHouse/ClickHouse/pull/16865) ([filimonov](https://github.com/filimonov)). +* Fix possible error `Illegal type of argument` for queries with `ORDER BY`. Fixes [#16580](https://github.com/ClickHouse/ClickHouse/issues/16580). [#16928](https://github.com/ClickHouse/ClickHouse/pull/16928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Install script should always create subdirs in config folders. This is only relevant for Docker build with custom config. [#16936](https://github.com/ClickHouse/ClickHouse/pull/16936) ([filimonov](https://github.com/filimonov)). +* Blame info was not calculated correctly in `clickhouse-git-import`. [#16959](https://github.com/ClickHouse/ClickHouse/pull/16959) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Reresolve the IP of the `format_avro_schema_registry_url` in case of errors. [#16985](https://github.com/ClickHouse/ClickHouse/pull/16985) ([filimonov](https://github.com/filimonov)). +* Fixed wrong result in big integers (128, 256 bit) when casting from double. [#16986](https://github.com/ClickHouse/ClickHouse/pull/16986) ([Mike Kot](https://github.com/myrrc)). +* Avoid unnecessary network errors for remote queries which may be cancelled while execution, like queries with `LIMIT`. [#17006](https://github.com/ClickHouse/ClickHouse/pull/17006) ([Azat Khuzhin](https://github.com/azat)). +* Fix LLVM's libunwind in the case when CFA register is RAX. This is the [bug](https://bugs.llvm.org/show_bug.cgi?id=48186) in [LLVM's libunwind](https://github.com/llvm/llvm-project/tree/master/libunwind). We already have workarounds for this bug. [#17046](https://github.com/ClickHouse/ClickHouse/pull/17046) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed crash on `CREATE TABLE ... AS some_table` query when `some_table` was created `AS table_function()` Fixes [#16944](https://github.com/ClickHouse/ClickHouse/issues/16944). [#17072](https://github.com/ClickHouse/ClickHouse/pull/17072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix bug when `ON CLUSTER` queries may hang forever for non-leader ReplicatedMergeTreeTables. [#17089](https://github.com/ClickHouse/ClickHouse/pull/17089) ([alesapin](https://github.com/alesapin)). +* fixes [#16923](https://github.com/ClickHouse/ClickHouse/issues/16923) fixes [#15883](https://github.com/ClickHouse/ClickHouse/issues/15883) Fix MaterializeMySQL SYNC failure when the modify MySQL binlog_checksum. [#17091](https://github.com/ClickHouse/ClickHouse/pull/17091) ([Winter Zhang](https://github.com/zhang2014)). +* Improve adaptive index granularity calculation when incoming blocks of data differ in bytes size a lot. [#17120](https://github.com/ClickHouse/ClickHouse/pull/17120) ([alesapin](https://github.com/alesapin)). +* Fix ColumnConst comparison which leads to crash. This fixed [#17088](https://github.com/ClickHouse/ClickHouse/issues/17088) . [#17135](https://github.com/ClickHouse/ClickHouse/pull/17135) ([Amos Bird](https://github.com/amosbird)). +* fix `toInt256(inf)` stack overflow. close [#17235](https://github.com/ClickHouse/ClickHouse/issues/17235). [#17257](https://github.com/ClickHouse/ClickHouse/pull/17257) ([flynn](https://github.com/ucasfl)). + +#### Build/Testing/Packaging Improvement +* Fix UBSan report when trying to convert infinite floating point number to integer. This closes [#14190](https://github.com/ClickHouse/ClickHouse/issues/14190). [#16677](https://github.com/ClickHouse/ClickHouse/pull/16677) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in cache dictionaries. This closes [#12641](https://github.com/ClickHouse/ClickHouse/issues/12641). [#16763](https://github.com/ClickHouse/ClickHouse/pull/16763) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not instrument 3rd-party libraries with UBSan. [#16764](https://github.com/ClickHouse/ClickHouse/pull/16764) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in Poco. This closes [#12719](https://github.com/ClickHouse/ClickHouse/issues/12719). [#16765](https://github.com/ClickHouse/ClickHouse/pull/16765) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix query_db_generate build error. [#16859](https://github.com/ClickHouse/ClickHouse/pull/16859) ([hhhhhzhen](https://github.com/su-houzhen)). +* Try fix fasttest submodule clone [#16132](https://github.com/ClickHouse/ClickHouse/issues/16132) https://clickhouse-test-reports.s3.yandex.net/16132/ad569f6d1bd2ce545db280daf7fbb9b8335de87b/fast_test.html#fail1. [#16908](https://github.com/ClickHouse/ClickHouse/pull/16908) ([Winter Zhang](https://github.com/zhang2014)). +* Fixing unstable test in tests/testflows/ldap/external_user_directory/tests/authentications.py. [#17161](https://github.com/ClickHouse/ClickHouse/pull/17161) ([vzakaznikov](https://github.com/vzakaznikov)). +* bump up rocksdb version to v6.14.5. [#17179](https://github.com/ClickHouse/ClickHouse/pull/17179) ([sundyli](https://github.com/sundy-li)). +* Update embedded timezone data to version 2020d (also update cctz to the latest master). [#17204](https://github.com/ClickHouse/ClickHouse/pull/17204) ([filimonov](https://github.com/filimonov)). +* Improvements in coverage building images. [#17233](https://github.com/ClickHouse/ClickHouse/pull/17233) ([alesapin](https://github.com/alesapin)). +* `std::logic_error` is used at line 294 of `base/common/StringRef.h`, so the appropriate `` header is required. [#17256](https://github.com/ClickHouse/ClickHouse/pull/17256) ([Matwey V. Kornilov](https://github.com/matwey)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'sync MySQL DDL atomicly'. [#16704](https://github.com/ClickHouse/ClickHouse/pull/16704) ([TCeason](https://github.com/TCeason)). +* NO CL ENTRY: 'RBAC Testflows - Server log intrumentation for debug and new ALTER tests'. [#16719](https://github.com/ClickHouse/ClickHouse/pull/16719) ([MyroTk](https://github.com/MyroTk)). +* NO CL ENTRY: 'Enabling existing testflows RBAC tests.'. [#16773](https://github.com/ClickHouse/ClickHouse/pull/16773) ([MyroTk](https://github.com/MyroTk)). +* NO CL ENTRY: 'Bump protobuf from 3.13.0 to 3.14.0 in /docs/tools'. [#17056](https://github.com/ClickHouse/ClickHouse/pull/17056) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Fixed a problem with the translation of the document'. [#17218](https://github.com/ClickHouse/ClickHouse/pull/17218) ([qianmoQ](https://github.com/qianmoQ)). + diff --git a/docs/changelogs/v20.12.2.1-stable.md b/docs/changelogs/v20.12.2.1-stable.md new file mode 100644 index 00000000000..6d8d6b151e5 --- /dev/null +++ b/docs/changelogs/v20.12.2.1-stable.md @@ -0,0 +1,127 @@ +### ClickHouse release v20.12.2.1-stable FIXME as compared to v20.11.1.5109-prestable + +#### Backward Incompatible Change +* Accept user settings related to file formats (e.g. `format_csv_delimiter`) in the `SETTINGS` clause when creating a table that uses `File` engine, and use these settings in all `INSERT`s and `SELECT`s. The file format settings changed in the current user session, or in the `SETTINGS` clause of a DML query itself, no longer affect the query. [#16591](https://github.com/ClickHouse/ClickHouse/pull/16591) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Enable `use_compact_format_in_distributed_parts_names` by default (see the documentation for the reference). [#16728](https://github.com/ClickHouse/ClickHouse/pull/16728) ([Azat Khuzhin](https://github.com/azat)). + +#### New Feature +* Added new ALTER UPDATE/DELETE IN PARTITION syntax. [#13403](https://github.com/ClickHouse/ClickHouse/pull/13403) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Add StorageEmbeddedRocksdb Engine. [#15073](https://github.com/ClickHouse/ClickHouse/pull/15073) ([sundyli](https://github.com/sundy-li)). +* Introduce the query `ALTER TABLE ... DROP|DETACH PART 'part_name'`. [#15511](https://github.com/ClickHouse/ClickHouse/pull/15511) ([nvartolomei](https://github.com/nvartolomei)). +* Make it possible to change the path to history file in `clickhouse-client` using the `--history_file` parameter. [#15960](https://github.com/ClickHouse/ClickHouse/pull/15960) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated DateTime, DateTime64 formatting to accept string Date literal format. [#16040](https://github.com/ClickHouse/ClickHouse/pull/16040) ([Maksim Kita](https://github.com/kitaisreal)). +* Add setting `aggregate_functions_null_for_empty`, this option will rewrite all aggregate functions in a query, adding -OrNull suffix to them. fix [10273](https://github.com/ClickHouse/ClickHouse/issues/10273). [#16123](https://github.com/ClickHouse/ClickHouse/pull/16123) ([flynn](https://github.com/ucasfl)). +* Add COLLATE support for Nullable, LowCardinality, Array and Tuple, where nested type is String. Also refactor the code associated with collations in ColumnString.cpp. [#16273](https://github.com/ClickHouse/ClickHouse/pull/16273) ([Kruglov Pavel](https://github.com/Avogar)). +* Possibility to distribute the merges between different replicas. Introduces the `execute_merges_on_single_replica_time_threshold` mergetree setting. [#16424](https://github.com/ClickHouse/ClickHouse/pull/16424) ([filimonov](https://github.com/filimonov)). +* add `*.xz` compression/decompression support.It enables using `*.xz` in `file()` function.This closes [#8828](https://github.com/ClickHouse/ClickHouse/issues/8828). [#16578](https://github.com/ClickHouse/ClickHouse/pull/16578) ([Abi Palagashvili](https://github.com/fibersel)). +* Add new `cmath` functions: - acosh - asinh - atan2 - atanh - cosh - hypot - log1p - sinh. [#16636](https://github.com/ClickHouse/ClickHouse/pull/16636) ([Konstantin Malanchev](https://github.com/hombit)). +* Add a possibility to input enum value as it's id in TSV and CSV formats by default. [#16834](https://github.com/ClickHouse/ClickHouse/pull/16834) ([Kruglov Pavel](https://github.com/Avogar)). +* New tcpPort() function returns TCP port listened by this server. [#17134](https://github.com/ClickHouse/ClickHouse/pull/17134) ([Ivan](https://github.com/abyss7)). + +#### Performance Improvement +* Now we can safely prune partitions with exact match. Useful case: Suppose table is partitioned by intHash64(x) % 100 and the query has condition on intHash64(x) % 100 verbatim, not on x. [#16253](https://github.com/ClickHouse/ClickHouse/pull/16253) ([Amos Bird](https://github.com/amosbird)). +* Use Floyd-Rivest algorithm, it should be the best for the ClickHouse use case of partial sorting. Bechmarks are in https://github.com/danlark1/miniselect and [here](https://drive.google.com/drive/folders/1DHEaeXgZuX6AJ9eByeZ8iQVQv0ueP8XM). [#16825](https://github.com/ClickHouse/ClickHouse/pull/16825) ([Daniel Kutenin](https://github.com/danlark1)). +* Backported in [#17589](https://github.com/ClickHouse/ClickHouse/issues/17589): Fix performance of reading from `Merge` tables over huge number of `MergeTree` tables. Fixes [#7748](https://github.com/ClickHouse/ClickHouse/issues/7748). [#16988](https://github.com/ClickHouse/ClickHouse/pull/16988) ([Anton Popov](https://github.com/CurtizJ)). + +#### Improvement +* Add `VIEW` subquery description to `EXPLAIN`. Limit push down optimisation for `VIEW`. Add local replicas of `Distributed` to query plan. [#14936](https://github.com/ClickHouse/ClickHouse/pull/14936) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Made `indexOf()` use BloomFilter. [#14977](https://github.com/ClickHouse/ClickHouse/pull/14977) ([achimbab](https://github.com/achimbab)). +* Throw exception about right sync privileges when MySQL sync user has error privileges. [#15977](https://github.com/ClickHouse/ClickHouse/pull/15977) ([TCeason](https://github.com/TCeason)). +* Fix possible stack overflow if a loop of materialized views is created. This closes [#15732](https://github.com/ClickHouse/ClickHouse/issues/15732). [#16048](https://github.com/ClickHouse/ClickHouse/pull/16048) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support SNI in https connections to remote resources. This will allow to connect to Cloudflare servers that require SNI. This fixes [#10055](https://github.com/ClickHouse/ClickHouse/issues/10055). [#16252](https://github.com/ClickHouse/ClickHouse/pull/16252) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now `ReplicatedMergeTree` tree engines family uses a separate thread pool for replicated fetches. Size of the pool limited by setting `background_fetches_pool_size` which can be tuned with a server restart. The default value of the setting is 3 and it means that the maximum amount of parallel fetches is equal to 3 (and it allows to utilize 10G network). Fixes #520. [#16390](https://github.com/ClickHouse/ClickHouse/pull/16390) ([alesapin](https://github.com/alesapin)). +* Now, `` configuration can be changed in `config.xml` and reloaded without server startup. [#16627](https://github.com/ClickHouse/ClickHouse/pull/16627) ([Amos Bird](https://github.com/amosbird)). +* Allow reinterpret between integers and floats of the same size. fix [16640](https://github.com/ClickHouse/ClickHouse/issues/16640). [#16657](https://github.com/ClickHouse/ClickHouse/pull/16657) ([flynn](https://github.com/ucasfl)). +* Workaround for use S3 with nginx server as proxy. Nginx currenty does not accept urls with empty path like `http://domain.com?delete`, but vanilla aws-sdk-cpp produces this kind of urls. This commit uses patched aws-sdk-cpp version, which makes urls with "/" as path in this cases, like `http://domain.com/?delete`. [#16709](https://github.com/ClickHouse/ClickHouse/pull/16709) ([ianton-ru](https://github.com/ianton-ru)). +* Remove empty directories for async INSERT at start of Distributed engine. [#16729](https://github.com/ClickHouse/ClickHouse/pull/16729) ([Azat Khuzhin](https://github.com/azat)). +* Usability improvement: better suggestions in syntax error message when `CODEC` expression is misplaced in `CREATE TABLE` query. This fixes [#12493](https://github.com/ClickHouse/ClickHouse/issues/12493). [#16768](https://github.com/ClickHouse/ClickHouse/pull/16768) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better exception message when configuration for distributed DDL is absent. This fixes [#5075](https://github.com/ClickHouse/ClickHouse/issues/5075). [#16769](https://github.com/ClickHouse/ClickHouse/pull/16769) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Apply `use_compact_format_in_distributed_parts_names` for each INSERT (with internal_replication). [#16788](https://github.com/ClickHouse/ClickHouse/pull/16788) ([Azat Khuzhin](https://github.com/azat)). +* Server refused to startup with exception message if wrong config is given (`metric_log`.`collect_interval_milliseconds` is missing). [#16815](https://github.com/ClickHouse/ClickHouse/pull/16815) ([Ivan](https://github.com/abyss7)). +* Add cutToFirstSignificantSubdomainWithWWW(). [#16845](https://github.com/ClickHouse/ClickHouse/pull/16845) ([Azat Khuzhin](https://github.com/azat)). +* Throw an informative error message when doing ATTACH/DETACH TABLE . Before this PR, `detach table ` works but leads to an ill-formed in-memory metadata. [#16885](https://github.com/ClickHouse/ClickHouse/pull/16885) ([Amos Bird](https://github.com/amosbird)). +* Remove empty parts after they were pruned by TTL, mutation, or collapsing merge algorithm. [#16895](https://github.com/ClickHouse/ClickHouse/pull/16895) ([Anton Popov](https://github.com/CurtizJ)). +* Make it possible to connect to `clickhouse-server` secure endpoint which requires SNI. This is possible when `clickhouse-server` is hosted behind TLS proxy. [#16938](https://github.com/ClickHouse/ClickHouse/pull/16938) ([filimonov](https://github.com/filimonov)). +* Set default `host` and `port` parameters for `SOURCE(CLICKHOUSE(...))` to current instance and set default `user` value to `'default'`. [#16997](https://github.com/ClickHouse/ClickHouse/pull/16997) ([Vladimir C](https://github.com/vdimir)). +* Add ability to output all rows as a JSON array in the `JSONEachRow` format, controlled by the `output_format_json_array_of_rows` setting. [#17152](https://github.com/ClickHouse/ClickHouse/pull/17152) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow formatting named tuples as JSON objects when using JSON input/output formats, controlled by the `output_format_json_named_tuples_as_objects` setting, disabled by default. [#17175](https://github.com/ClickHouse/ClickHouse/pull/17175) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Correct grammar in error message in JSONEachRow, JSONCompactEachRow, and RegexpRow input formats. [#17205](https://github.com/ClickHouse/ClickHouse/pull/17205) ([nico piderman](https://github.com/sneako)). + +#### Bug Fix +* Backported in [#17620](https://github.com/ClickHouse/ClickHouse/issues/17620): Throw error when use ColumnTransformer replace non exist column. [#16183](https://github.com/ClickHouse/ClickHouse/pull/16183) ([hexiaoting](https://github.com/hexiaoting)). +* fixes [#16574](https://github.com/ClickHouse/ClickHouse/issues/16574) fixes [#16231](https://github.com/ClickHouse/ClickHouse/issues/16231) fix remote query failure when using 'if' suffix aggregate function. [#16610](https://github.com/ClickHouse/ClickHouse/pull/16610) ([Winter Zhang](https://github.com/zhang2014)). +* Fixed [#16081](https://github.com/ClickHouse/ClickHouse/issues/16081). [#16613](https://github.com/ClickHouse/ClickHouse/pull/16613) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* This will fix optimize_read_in_order/optimize_aggregation_in_order with max_threads>0 and expression in ORDER BY. [#16637](https://github.com/ClickHouse/ClickHouse/pull/16637) ([Azat Khuzhin](https://github.com/azat)). +* Fixed uncontrolled growth of TDigest. [#16680](https://github.com/ClickHouse/ClickHouse/pull/16680) ([hrissan](https://github.com/hrissan)). +* Backported in [#17315](https://github.com/ClickHouse/ClickHouse/issues/17315): Return number of affected rows for INSERT queries via MySQL protocol. Previously ClickHouse used to always return 0, it's fixed. Fixes [#16605](https://github.com/ClickHouse/ClickHouse/issues/16605). [#16715](https://github.com/ClickHouse/ClickHouse/pull/16715) ([Winter Zhang](https://github.com/zhang2014)). +* Turn off parallel parsing when there is no enough memory for all threads to work simultaneously. Also there could be exceptions like "Memory limit exceeded" when somebody will try to insert extremely huge rows (> min_chunk_bytes_for_parallel_parsing), because each piece to parse has to be independent set of strings (one or more). [#16721](https://github.com/ClickHouse/ClickHouse/pull/16721) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix `IN` operator over several columns and tuples with enabled `transform_null_in` setting. Fixes [#15310](https://github.com/ClickHouse/ClickHouse/issues/15310). [#16722](https://github.com/ClickHouse/ClickHouse/pull/16722) ([Anton Popov](https://github.com/CurtizJ)). +* Mask password in data_path in the system.distribution_queue. [#16727](https://github.com/ClickHouse/ClickHouse/pull/16727) ([Azat Khuzhin](https://github.com/azat)). +* Not for changelog. [#16757](https://github.com/ClickHouse/ClickHouse/pull/16757) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17520](https://github.com/ClickHouse/ClickHouse/issues/17520): Fix optimize_trivial_count_query with partition predicate. [#16767](https://github.com/ClickHouse/ClickHouse/pull/16767) ([Azat Khuzhin](https://github.com/azat)). +* If no memory can be allocated while writing table metadata on disk, broken metadata file can be written. [#16772](https://github.com/ClickHouse/ClickHouse/pull/16772) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix crash when using `any` without any arguments. This is for [#16803](https://github.com/ClickHouse/ClickHouse/issues/16803) . cc @azat. [#16826](https://github.com/ClickHouse/ClickHouse/pull/16826) ([Amos Bird](https://github.com/amosbird)). +* Abort multipart upload if no data was written to WriteBufferFromS3. [#16840](https://github.com/ClickHouse/ClickHouse/pull/16840) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix rare silent crashes when query profiler is on and ClickHouse is installed on OS with glibc version that has (supposedly) broken asynchronous unwind tables for some functions. This fixes [#15301](https://github.com/ClickHouse/ClickHouse/issues/15301). This fixes [#13098](https://github.com/ClickHouse/ClickHouse/issues/13098). [#16846](https://github.com/ClickHouse/ClickHouse/pull/16846) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Prevent clickhouse server crashes when using TimeSeriesGroupSum. [#16865](https://github.com/ClickHouse/ClickHouse/pull/16865) ([filimonov](https://github.com/filimonov)). +* Backported in [#17340](https://github.com/ClickHouse/ClickHouse/issues/17340): TODO. [#16866](https://github.com/ClickHouse/ClickHouse/pull/16866) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix possible error `Illegal type of argument` for queries with `ORDER BY`. Fixes [#16580](https://github.com/ClickHouse/ClickHouse/issues/16580). [#16928](https://github.com/ClickHouse/ClickHouse/pull/16928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Install script should always create subdirs in config folders. This is only relevant for Docker build with custom config. [#16936](https://github.com/ClickHouse/ClickHouse/pull/16936) ([filimonov](https://github.com/filimonov)). +* Backported in [#17586](https://github.com/ClickHouse/ClickHouse/issues/17586): Fix optimization of group by with enabled setting `optimize_aggregators_of_group_by_keys` and joins. Fixes [#12604](https://github.com/ClickHouse/ClickHouse/issues/12604). [#16951](https://github.com/ClickHouse/ClickHouse/pull/16951) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17593](https://github.com/ClickHouse/ClickHouse/issues/17593): Fix order by optimization with monotonous functions. Fixes [#16107](https://github.com/ClickHouse/ClickHouse/issues/16107). [#16956](https://github.com/ClickHouse/ClickHouse/pull/16956) ([Anton Popov](https://github.com/CurtizJ)). +* Blame info was not calculated correctly in `clickhouse-git-import`. [#16959](https://github.com/ClickHouse/ClickHouse/pull/16959) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Reresolve the IP of the `format_avro_schema_registry_url` in case of errors. [#16985](https://github.com/ClickHouse/ClickHouse/pull/16985) ([filimonov](https://github.com/filimonov)). +* Fixed wrong result in big integers (128, 256 bit) when casting from double. [#16986](https://github.com/ClickHouse/ClickHouse/pull/16986) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#17338](https://github.com/ClickHouse/ClickHouse/issues/17338): Fix Merge(Distributed()) with JOIN. [#16993](https://github.com/ClickHouse/ClickHouse/pull/16993) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#17746](https://github.com/ClickHouse/ClickHouse/issues/17746): - Fix optimize_distributed_group_by_sharding_key for query with OFFSET only. [#16996](https://github.com/ClickHouse/ClickHouse/pull/16996) ([Azat Khuzhin](https://github.com/azat)). +* Avoid unnecessary network errors for remote queries which may be cancelled while execution, like queries with `LIMIT`. [#17006](https://github.com/ClickHouse/ClickHouse/pull/17006) ([Azat Khuzhin](https://github.com/azat)). +* Fix LLVM's libunwind in the case when CFA register is RAX. This is the [bug](https://bugs.llvm.org/show_bug.cgi?id=48186) in [LLVM's libunwind](https://github.com/llvm/llvm-project/tree/master/libunwind). We already have workarounds for this bug. [#17046](https://github.com/ClickHouse/ClickHouse/pull/17046) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17429](https://github.com/ClickHouse/ClickHouse/issues/17429): Bug fix for funciton fuzzBits, related issue: [#16980](https://github.com/ClickHouse/ClickHouse/issues/16980). [#17051](https://github.com/ClickHouse/ClickHouse/pull/17051) ([hexiaoting](https://github.com/hexiaoting)). +* Fixed crash on `CREATE TABLE ... AS some_table` query when `some_table` was created `AS table_function()` Fixes [#16944](https://github.com/ClickHouse/ClickHouse/issues/16944). [#17072](https://github.com/ClickHouse/ClickHouse/pull/17072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix bug when `ON CLUSTER` queries may hang forever for non-leader ReplicatedMergeTreeTables. [#17089](https://github.com/ClickHouse/ClickHouse/pull/17089) ([alesapin](https://github.com/alesapin)). +* fixes [#16923](https://github.com/ClickHouse/ClickHouse/issues/16923) fixes [#15883](https://github.com/ClickHouse/ClickHouse/issues/15883) Fix MaterializeMySQL SYNC failure when the modify MySQL binlog_checksum. [#17091](https://github.com/ClickHouse/ClickHouse/pull/17091) ([Winter Zhang](https://github.com/zhang2014)). +* Improve adaptive index granularity calculation when incoming blocks of data differ in bytes size a lot. [#17120](https://github.com/ClickHouse/ClickHouse/pull/17120) ([alesapin](https://github.com/alesapin)). +* Fix ColumnConst comparison which leads to crash. This fixed [#17088](https://github.com/ClickHouse/ClickHouse/issues/17088) . [#17135](https://github.com/ClickHouse/ClickHouse/pull/17135) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17560](https://github.com/ClickHouse/ClickHouse/issues/17560): Fix possible wrong index analysis when the types of the index comparison are different. This fixes [#17122](https://github.com/ClickHouse/ClickHouse/issues/17122). [#17145](https://github.com/ClickHouse/ClickHouse/pull/17145) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17394](https://github.com/ClickHouse/ClickHouse/issues/17394): Fix [#15235](https://github.com/ClickHouse/ClickHouse/issues/15235). When clickhouse-copier handle non-partitioned table, throws segfault error. [#17248](https://github.com/ClickHouse/ClickHouse/pull/17248) ([Qi Chen](https://github.com/kaka11chen)). +* Backported in [#17406](https://github.com/ClickHouse/ClickHouse/issues/17406): Fix set index invalidation when there are const columns in the subquery. This fixes [#17246](https://github.com/ClickHouse/ClickHouse/issues/17246) . [#17249](https://github.com/ClickHouse/ClickHouse/pull/17249) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17422](https://github.com/ClickHouse/ClickHouse/issues/17422): Fix possible `Unexpected packet Data received from client` error for Distributed queries with `LIMIT`. [#17254](https://github.com/ClickHouse/ClickHouse/pull/17254) ([Azat Khuzhin](https://github.com/azat)). +* fix `toInt256(inf)` stack overflow. close [#17235](https://github.com/ClickHouse/ClickHouse/issues/17235). [#17257](https://github.com/ClickHouse/ClickHouse/pull/17257) ([flynn](https://github.com/ucasfl)). +* Backported in [#17489](https://github.com/ClickHouse/ClickHouse/issues/17489): Fix crash while reading from `JOIN` table with `LowCardinality` types. Fixes [#17228](https://github.com/ClickHouse/ClickHouse/issues/17228). [#17397](https://github.com/ClickHouse/ClickHouse/pull/17397) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17451](https://github.com/ClickHouse/ClickHouse/issues/17451): Fixed high CPU usage in background tasks of *MergeTree tables. [#17416](https://github.com/ClickHouse/ClickHouse/pull/17416) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17493](https://github.com/ClickHouse/ClickHouse/issues/17493): Fix duplicates after `DISTINCT` which were possible because of incorrect optimization. Fixes [#17294](https://github.com/ClickHouse/ClickHouse/issues/17294). [#17296](https://github.com/ClickHouse/ClickHouse/pull/17296) ([li chengxiang](https://github.com/chengxianglibra)). [#17439](https://github.com/ClickHouse/ClickHouse/pull/17439) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17523](https://github.com/ClickHouse/ClickHouse/issues/17523): Fix `ORDER BY` with enabled setting `optimize_redundant_functions_in_order_by`. [#17471](https://github.com/ClickHouse/ClickHouse/pull/17471) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17533](https://github.com/ClickHouse/ClickHouse/issues/17533): Fix bug when mark cache size was underestimated by clickhouse. It may happen when there are a lot of tiny files with marks. [#17496](https://github.com/ClickHouse/ClickHouse/pull/17496) ([alesapin](https://github.com/alesapin)). +* Backported in [#17625](https://github.com/ClickHouse/ClickHouse/issues/17625): Fix alter query hang when the corresponding mutation was killed on the different replica. Fixes [#16953](https://github.com/ClickHouse/ClickHouse/issues/16953). [#17499](https://github.com/ClickHouse/ClickHouse/pull/17499) ([alesapin](https://github.com/alesapin)). +* Backported in [#17611](https://github.com/ClickHouse/ClickHouse/issues/17611): Fix the issue when server can stop accepting connections in very rare cases. [#17542](https://github.com/ClickHouse/ClickHouse/pull/17542) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17607](https://github.com/ClickHouse/ClickHouse/issues/17607): When clickhouse-client is used in interactive mode with multiline queries, single line comment was erronously extended till the end of query. This fixes [#13654](https://github.com/ClickHouse/ClickHouse/issues/13654). [#17565](https://github.com/ClickHouse/ClickHouse/pull/17565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17683](https://github.com/ClickHouse/ClickHouse/issues/17683): Exception `fmt::v7::format_error` can be logged in background for MergeTree tables. This fixes [#17613](https://github.com/ClickHouse/ClickHouse/issues/17613). [#17615](https://github.com/ClickHouse/ClickHouse/pull/17615) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17697](https://github.com/ClickHouse/ClickHouse/issues/17697): In might be determined incorrectly if cluster is circular- (cross-) replicated or not when executing `ON CLUSTER` query due to race condition when `pool_size` > 1. It's fixed. [#17640](https://github.com/ClickHouse/ClickHouse/pull/17640) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17727](https://github.com/ClickHouse/ClickHouse/issues/17727): Fixed `Function not implemented` error when executing `RENAME` query in `Atomic` database with ClickHouse running on Windows Subsystem for Linux. Fixes [#17661](https://github.com/ClickHouse/ClickHouse/issues/17661). [#17664](https://github.com/ClickHouse/ClickHouse/pull/17664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17784](https://github.com/ClickHouse/ClickHouse/issues/17784): Fixed problem when ClickHouse fails to resume connection to MySQL servers. [#17681](https://github.com/ClickHouse/ClickHouse/pull/17681) ([Alexander Kazakov](https://github.com/Akazz)). +* Backported in [#17780](https://github.com/ClickHouse/ClickHouse/issues/17780): Exception message about max table size to drop was displayed incorrectly. [#17764](https://github.com/ClickHouse/ClickHouse/pull/17764) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17817](https://github.com/ClickHouse/ClickHouse/issues/17817): Do not restore parts from WAL if `in_memory_parts_enable_wal` is disabled. [#17802](https://github.com/ClickHouse/ClickHouse/pull/17802) ([detailyang](https://github.com/detailyang)). +* Backported in [#17842](https://github.com/ClickHouse/ClickHouse/issues/17842): fix incorrect initialize `max_compress_block_size` of MergeTreeWriterSettings with `min_compress_block_size`. [#17833](https://github.com/ClickHouse/ClickHouse/pull/17833) ([flynn](https://github.com/ucasfl)). + +#### Build/Testing/Packaging Improvement +* Fix UBSan report when trying to convert infinite floating point number to integer. This closes [#14190](https://github.com/ClickHouse/ClickHouse/issues/14190). [#16677](https://github.com/ClickHouse/ClickHouse/pull/16677) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in cache dictionaries. This closes [#12641](https://github.com/ClickHouse/ClickHouse/issues/12641). [#16763](https://github.com/ClickHouse/ClickHouse/pull/16763) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not instrument 3rd-party libraries with UBSan. [#16764](https://github.com/ClickHouse/ClickHouse/pull/16764) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in Poco. This closes [#12719](https://github.com/ClickHouse/ClickHouse/issues/12719). [#16765](https://github.com/ClickHouse/ClickHouse/pull/16765) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix query_db_generate build error. [#16859](https://github.com/ClickHouse/ClickHouse/pull/16859) ([hhhhhzhen](https://github.com/su-houzhen)). +* Try fix fasttest submodule clone [#16132](https://github.com/ClickHouse/ClickHouse/issues/16132) https://clickhouse-test-reports.s3.yandex.net/16132/ad569f6d1bd2ce545db280daf7fbb9b8335de87b/fast_test.html#fail1. [#16908](https://github.com/ClickHouse/ClickHouse/pull/16908) ([Winter Zhang](https://github.com/zhang2014)). +* Fixing unstable test in tests/testflows/ldap/external_user_directory/tests/authentications.py. [#17161](https://github.com/ClickHouse/ClickHouse/pull/17161) ([vzakaznikov](https://github.com/vzakaznikov)). +* bump up rocksdb version to v6.14.5. [#17179](https://github.com/ClickHouse/ClickHouse/pull/17179) ([sundyli](https://github.com/sundy-li)). +* Update embedded timezone data to version 2020d (also update cctz to the latest master). [#17204](https://github.com/ClickHouse/ClickHouse/pull/17204) ([filimonov](https://github.com/filimonov)). +* Improvements in coverage building images. [#17233](https://github.com/ClickHouse/ClickHouse/pull/17233) ([alesapin](https://github.com/alesapin)). +* `std::logic_error` is used at line 294 of `base/common/StringRef.h`, so the appropriate `` header is required. [#17256](https://github.com/ClickHouse/ClickHouse/pull/17256) ([Matwey V. Kornilov](https://github.com/matwey)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'sync MySQL DDL atomicly'. [#16704](https://github.com/ClickHouse/ClickHouse/pull/16704) ([TCeason](https://github.com/TCeason)). +* NO CL ENTRY: 'RBAC Testflows - Server log intrumentation for debug and new ALTER tests'. [#16719](https://github.com/ClickHouse/ClickHouse/pull/16719) ([MyroTk](https://github.com/MyroTk)). +* NO CL ENTRY: 'Enabling existing testflows RBAC tests.'. [#16773](https://github.com/ClickHouse/ClickHouse/pull/16773) ([MyroTk](https://github.com/MyroTk)). +* NO CL ENTRY: 'Bump protobuf from 3.13.0 to 3.14.0 in /docs/tools'. [#17056](https://github.com/ClickHouse/ClickHouse/pull/17056) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Fixed a problem with the translation of the document'. [#17218](https://github.com/ClickHouse/ClickHouse/pull/17218) ([qianmoQ](https://github.com/qianmoQ)). + diff --git a/docs/changelogs/v20.12.3.3-stable.md b/docs/changelogs/v20.12.3.3-stable.md new file mode 100644 index 00000000000..9056ddd8b37 --- /dev/null +++ b/docs/changelogs/v20.12.3.3-stable.md @@ -0,0 +1,2 @@ +### ClickHouse release v20.12.3.3-stable FIXME as compared to v20.12.2.1-stable + diff --git a/docs/changelogs/v20.12.4.5-stable.md b/docs/changelogs/v20.12.4.5-stable.md new file mode 100644 index 00000000000..b28c5462c9d --- /dev/null +++ b/docs/changelogs/v20.12.4.5-stable.md @@ -0,0 +1,14 @@ +### ClickHouse release v20.12.4.5-stable FIXME as compared to v20.12.3.3-stable + +#### Bug Fix +* Backported in [#18392](https://github.com/ClickHouse/ClickHouse/issues/18392): Fix empty `system.stack_trace` table when server is running in daemon mode. [#17630](https://github.com/ClickHouse/ClickHouse/pull/17630) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#18043](https://github.com/ClickHouse/ClickHouse/issues/18043): Fix possible segfault in `topK` aggregate function. This closes [#17404](https://github.com/ClickHouse/ClickHouse/issues/17404). [#17845](https://github.com/ClickHouse/ClickHouse/pull/17845) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#18111](https://github.com/ClickHouse/ClickHouse/issues/18111): Fix max_distributed_connections (affects `prefer_localhost_replica=1` and `max_threads!=max_distributed_connections`). [#17848](https://github.com/ClickHouse/ClickHouse/pull/17848) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#18021](https://github.com/ClickHouse/ClickHouse/issues/18021): Trivial query optimization was producing wrong result if query contains ARRAY JOIN (so query is actually non trivial). [#17887](https://github.com/ClickHouse/ClickHouse/pull/17887) ([sundyli](https://github.com/sundy-li)). +* Backported in [#17981](https://github.com/ClickHouse/ClickHouse/issues/17981): fixes [#15187](https://github.com/ClickHouse/ClickHouse/issues/15187) fixes [#17912](https://github.com/ClickHouse/ClickHouse/issues/17912) support convert MySQL prefix index for MaterializeMySQL CC: @tavplubix. [#17944](https://github.com/ClickHouse/ClickHouse/pull/17944) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#18078](https://github.com/ClickHouse/ClickHouse/issues/18078): Fixed `std::out_of_range: basic_string` in S3 URL parsing. [#18059](https://github.com/ClickHouse/ClickHouse/pull/18059) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#18179](https://github.com/ClickHouse/ClickHouse/issues/18179): Fix `Unknown setting profile` error on attempt to set settings profile. [#18167](https://github.com/ClickHouse/ClickHouse/pull/18167) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#18359](https://github.com/ClickHouse/ClickHouse/issues/18359): fixes [#18186](https://github.com/ClickHouse/ClickHouse/issues/18186) fixes [#16372](https://github.com/ClickHouse/ClickHouse/issues/16372) fix unique key convert crash in MaterializeMySQL database engine. [#18211](https://github.com/ClickHouse/ClickHouse/pull/18211) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#18258](https://github.com/ClickHouse/ClickHouse/issues/18258): Fix key comparison between Enum and Int types. This fixes [#17989](https://github.com/ClickHouse/ClickHouse/issues/17989). [#18214](https://github.com/ClickHouse/ClickHouse/pull/18214) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#18296](https://github.com/ClickHouse/ClickHouse/issues/18296): - Fixed issue when `clickhouse-odbc-bridge` process is unreachable by server on machines with dual IPv4/IPv6 stack; - Fixed issue when ODBC dictionary updates are performed using malformed queries and/or cause crashes; Possibly closes [#14489](https://github.com/ClickHouse/ClickHouse/issues/14489). [#18278](https://github.com/ClickHouse/ClickHouse/pull/18278) ([Denis Glazachev](https://github.com/traceon)). + diff --git a/docs/changelogs/v20.12.5.14-stable.md b/docs/changelogs/v20.12.5.14-stable.md new file mode 100644 index 00000000000..04d5153a967 --- /dev/null +++ b/docs/changelogs/v20.12.5.14-stable.md @@ -0,0 +1,14 @@ +### ClickHouse release v20.12.5.14-stable FIXME as compared to v20.12.4.5-stable + +#### Bug Fix +* Backported in [#18166](https://github.com/ClickHouse/ClickHouse/issues/18166): Fix error when query `MODIFY COLUMN ... REMOVE TTL` doesn't actually remove column TTL. [#18130](https://github.com/ClickHouse/ClickHouse/pull/18130) ([alesapin](https://github.com/alesapin)). +* Backported in [#18424](https://github.com/ClickHouse/ClickHouse/issues/18424): Fix possible crashes in aggregate functions with combinator `Distinct`, while using two-level aggregation. Fixes [#17682](https://github.com/ClickHouse/ClickHouse/issues/17682). [#18365](https://github.com/ClickHouse/ClickHouse/pull/18365) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#18428](https://github.com/ClickHouse/ClickHouse/issues/18428): Fix filling table `system.settings_profile_elements`. This PR fixes [#18231](https://github.com/ClickHouse/ClickHouse/issues/18231). [#18379](https://github.com/ClickHouse/ClickHouse/pull/18379) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#18482](https://github.com/ClickHouse/ClickHouse/issues/18482): Restrict merges from wide to compact parts. In case of vertical merge it led to broken result part. [#18381](https://github.com/ClickHouse/ClickHouse/pull/18381) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#18471](https://github.com/ClickHouse/ClickHouse/issues/18471): Fixed `value is too short` error when executing `toType(...)` functions (`toDate`, `toUInt32`, etc) with argument of type `Nullable(String)`. Now such functions return `NULL` on parsing errors instead of throwing exception. Fixes [#7673](https://github.com/ClickHouse/ClickHouse/issues/7673). [#18445](https://github.com/ClickHouse/ClickHouse/pull/18445) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#18533](https://github.com/ClickHouse/ClickHouse/issues/18533): Proper support for 12AM in `parseDateTimeBestEffort` function. This fixes [#18402](https://github.com/ClickHouse/ClickHouse/issues/18402). [#18449](https://github.com/ClickHouse/ClickHouse/pull/18449) ([vladimir-golovchenko](https://github.com/vladimir-golovchenko)). +* Backported in [#18502](https://github.com/ClickHouse/ClickHouse/issues/18502): Disable write with AIO during merges because it can lead to extremely rare data corruption of primary key columns during merge. [#18481](https://github.com/ClickHouse/ClickHouse/pull/18481) ([alesapin](https://github.com/alesapin)). + +#### Build/Testing/Packaging Improvement +* Backported in [#18546](https://github.com/ClickHouse/ClickHouse/issues/18546): Update timezones info to 2020e. [#18531](https://github.com/ClickHouse/ClickHouse/pull/18531) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v20.12.5.18-stable.md b/docs/changelogs/v20.12.5.18-stable.md new file mode 100644 index 00000000000..ec9c79b5f4f --- /dev/null +++ b/docs/changelogs/v20.12.5.18-stable.md @@ -0,0 +1,57 @@ +### ClickHouse release v20.12.5.18-stable FIXME as compared to v20.12.5.14-stable + +#### Improvement +* Backported in [#19149](https://github.com/ClickHouse/ClickHouse/issues/19149): Explicitly set uid / gid of clickhouse user & group to the fixed values (101) in clickhouse-server images. [#19096](https://github.com/ClickHouse/ClickHouse/pull/19096) ([filimonov](https://github.com/filimonov)). + +#### Bug Fix +* Backported in [#18267](https://github.com/ClickHouse/ClickHouse/issues/18267): Fix indeterministic functions with predicate optimizer. This fixes [#17244](https://github.com/ClickHouse/ClickHouse/issues/17244). [#17273](https://github.com/ClickHouse/ClickHouse/pull/17273) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#19655](https://github.com/ClickHouse/ClickHouse/issues/19655): fix data type convert issue for mysql engine ... [#18124](https://github.com/ClickHouse/ClickHouse/pull/18124) ([bo zeng](https://github.com/mis98zb)). +* Backported in [#18311](https://github.com/ClickHouse/ClickHouse/issues/18311): Fix inserting a row with default value in case of parsing error in the last column. Fixes [#17712](https://github.com/ClickHouse/ClickHouse/issues/17712). [#18182](https://github.com/ClickHouse/ClickHouse/pull/18182) ([Jianmei Zhang](https://github.com/zhangjmruc)). +* Backported in [#18229](https://github.com/ClickHouse/ClickHouse/issues/18229): Fix possible incomplete query result while reading from `MergeTree*` in case of read backoff (message ` MergeTreeReadPool: Will lower number of threads` in logs). Was introduced in [#16423](https://github.com/ClickHouse/ClickHouse/issues/16423). Fixes [#18137](https://github.com/ClickHouse/ClickHouse/issues/18137). [#18216](https://github.com/ClickHouse/ClickHouse/pull/18216) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#18630](https://github.com/ClickHouse/ClickHouse/issues/18630): `SELECT JOIN` now requires the `SELECT` privilege on each of the joined tables. This PR fixes [#17654](https://github.com/ClickHouse/ClickHouse/issues/17654). [#18232](https://github.com/ClickHouse/ClickHouse/pull/18232) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#19202](https://github.com/ClickHouse/ClickHouse/issues/19202): `SELECT count() FROM table` now can be executed if only one any column can be selected from the `table`. This PR fixes [#10639](https://github.com/ClickHouse/ClickHouse/issues/10639). [#18233](https://github.com/ClickHouse/ClickHouse/pull/18233) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#19162](https://github.com/ClickHouse/ClickHouse/issues/19162): Fix index analysis of binary functions with constant argument which leads to wrong query results. This fixes [#18364](https://github.com/ClickHouse/ClickHouse/issues/18364). [#18373](https://github.com/ClickHouse/ClickHouse/pull/18373) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#19716](https://github.com/ClickHouse/ClickHouse/issues/19716): Disable constant folding for subqueries on the analysis stage, when the result cannot be calculated. [#18446](https://github.com/ClickHouse/ClickHouse/pull/18446) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#18601](https://github.com/ClickHouse/ClickHouse/issues/18601): Fix bug which may lead to `ALTER` queries hung after corresponding mutation kill. Found by thread fuzzer. [#18518](https://github.com/ClickHouse/ClickHouse/pull/18518) ([alesapin](https://github.com/alesapin)). +* Backported in [#18577](https://github.com/ClickHouse/ClickHouse/issues/18577): Fix possible `Pipeline stuck` error while using `ORDER BY` after subquery with `RIGHT` or `FULL` join. [#18550](https://github.com/ClickHouse/ClickHouse/pull/18550) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#18606](https://github.com/ClickHouse/ClickHouse/issues/18606): Add FixedString Data type support. I'll get this exception "Code: 50, e.displayText() = DB::Exception: Unsupported type FixedString(1)" when replicating data from MySQL to ClickHouse. This patch fixes bug [#18450](https://github.com/ClickHouse/ClickHouse/issues/18450) Also fixes [#6556](https://github.com/ClickHouse/ClickHouse/issues/6556). [#18553](https://github.com/ClickHouse/ClickHouse/pull/18553) ([awesomeleo](https://github.com/awesomeleo)). +* Backported in [#18643](https://github.com/ClickHouse/ClickHouse/issues/18643): Fix removing of empty parts in `ReplicatedMergeTree` tables, created with old syntax. Fixes [#18582](https://github.com/ClickHouse/ClickHouse/issues/18582). [#18614](https://github.com/ClickHouse/ClickHouse/pull/18614) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#18738](https://github.com/ClickHouse/ClickHouse/issues/18738): Fix Logger with unmatched arg size. [#18717](https://github.com/ClickHouse/ClickHouse/pull/18717) ([sundyli](https://github.com/sundy-li)). +* Backported in [#18949](https://github.com/ClickHouse/ClickHouse/issues/18949): Fixed `Attempt to read after eof` error when trying to `CAST` `NULL` from `Nullable(String)` to `Nullable(Decimal(P, S))`. Now function `CAST` returns `NULL` when it cannot parse decimal from nullable string. Fixes [#7690](https://github.com/ClickHouse/ClickHouse/issues/7690). [#18718](https://github.com/ClickHouse/ClickHouse/pull/18718) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#18802](https://github.com/ClickHouse/ClickHouse/issues/18802): Asynchronous distributed INSERTs can be rejected by the server if the setting `network_compression_method` is globally set to non-default value. This fixes [#18741](https://github.com/ClickHouse/ClickHouse/issues/18741). [#18776](https://github.com/ClickHouse/ClickHouse/pull/18776) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#18835](https://github.com/ClickHouse/ClickHouse/issues/18835): Fix *If combinator with unary function and Nullable types. [#18806](https://github.com/ClickHouse/ClickHouse/pull/18806) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#18909](https://github.com/ClickHouse/ClickHouse/issues/18909): Fix possible hang at shutdown in clickhouse-local. This fixes [#18891](https://github.com/ClickHouse/ClickHouse/issues/18891). [#18893](https://github.com/ClickHouse/ClickHouse/pull/18893) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19114](https://github.com/ClickHouse/ClickHouse/issues/19114): Attach partition should reset the mutation. [#18804](https://github.com/ClickHouse/ClickHouse/issues/18804). [#18935](https://github.com/ClickHouse/ClickHouse/pull/18935) ([fastio](https://github.com/fastio)). +* Backported in [#18967](https://github.com/ClickHouse/ClickHouse/issues/18967): Fix bug when mutation with some escaped text (like `ALTER ... UPDATE e = CAST('foo', 'Enum8(\'foo\' = 1')` serialized incorrectly. Fixes [#18878](https://github.com/ClickHouse/ClickHouse/issues/18878). [#18944](https://github.com/ClickHouse/ClickHouse/pull/18944) ([alesapin](https://github.com/alesapin)). +* Backported in [#19008](https://github.com/ClickHouse/ClickHouse/issues/19008): Fix incorrect behavior when `ALTER TABLE ... DROP PART 'part_name'` query removes all deduplication blocks for the whole partition. Fixes [#18874](https://github.com/ClickHouse/ClickHouse/issues/18874). [#18969](https://github.com/ClickHouse/ClickHouse/pull/18969) ([alesapin](https://github.com/alesapin)). +* Backported in [#19192](https://github.com/ClickHouse/ClickHouse/issues/19192): Fixed very rare deadlock at shutdown. [#18977](https://github.com/ClickHouse/ClickHouse/pull/18977) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19094](https://github.com/ClickHouse/ClickHouse/issues/19094): Disable `optimize_move_functions_out_of_any` because optimization is not always correct. This closes [#18051](https://github.com/ClickHouse/ClickHouse/issues/18051). This closes [#18973](https://github.com/ClickHouse/ClickHouse/issues/18973). [#18981](https://github.com/ClickHouse/ClickHouse/pull/18981) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19068](https://github.com/ClickHouse/ClickHouse/issues/19068): Join tries to materialize const columns, but our code waits for them in other places. [#18982](https://github.com/ClickHouse/ClickHouse/pull/18982) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#19051](https://github.com/ClickHouse/ClickHouse/issues/19051): Fix inserting of `LowCardinality` column to table with `TinyLog` engine. Fixes [#18629](https://github.com/ClickHouse/ClickHouse/issues/18629). [#19010](https://github.com/ClickHouse/ClickHouse/pull/19010) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19083](https://github.com/ClickHouse/ClickHouse/issues/19083): Fix possible error `Expected single dictionary argument for function` if use function `ignore` with `LowCardinality` argument. Fixes [#14275](https://github.com/ClickHouse/ClickHouse/issues/14275). [#19016](https://github.com/ClickHouse/ClickHouse/pull/19016) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19047](https://github.com/ClickHouse/ClickHouse/issues/19047): Make sure `groupUniqArray` returns correct type for argument of Enum type. This closes [#17875](https://github.com/ClickHouse/ClickHouse/issues/17875). [#19019](https://github.com/ClickHouse/ClickHouse/pull/19019) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19128](https://github.com/ClickHouse/ClickHouse/issues/19128): Restrict `MODIFY TTL` queries for `MergeTree` tables created in old syntax. Previously the query succeeded, but actually it had no effect. [#19064](https://github.com/ClickHouse/ClickHouse/pull/19064) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#19565](https://github.com/ClickHouse/ClickHouse/issues/19565): Fixed `There is no checkpoint` error when inserting data through http interface using `Template` or `CustomSeparated` format. Fixes [#19021](https://github.com/ClickHouse/ClickHouse/issues/19021). [#19072](https://github.com/ClickHouse/ClickHouse/pull/19072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19232](https://github.com/ClickHouse/ClickHouse/issues/19232): Fix startup bug when clickhouse was not able to read compression codec from `LowCardinality(Nullable(...))` and throws exception `Attempt to read after EOF`. Fixes [#18340](https://github.com/ClickHouse/ClickHouse/issues/18340). [#19101](https://github.com/ClickHouse/ClickHouse/pull/19101) ([alesapin](https://github.com/alesapin)). +* Backported in [#19180](https://github.com/ClickHouse/ClickHouse/issues/19180): Fix infinite reading from file in `ORC` format (was introduced in [#10580](https://github.com/ClickHouse/ClickHouse/issues/10580)). Fixes [#19095](https://github.com/ClickHouse/ClickHouse/issues/19095). [#19134](https://github.com/ClickHouse/ClickHouse/pull/19134) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19269](https://github.com/ClickHouse/ClickHouse/issues/19269): Fix bug when concurrent `ALTER` and `DROP` queries may hang while processing ReplicatedMergeTree table. [#19237](https://github.com/ClickHouse/ClickHouse/pull/19237) ([alesapin](https://github.com/alesapin)). +* Backported in [#19424](https://github.com/ClickHouse/ClickHouse/issues/19424): Fix error `Cannot convert column now64() because it is constant but values of constants are different in source and result`. Continuation of [#7156](https://github.com/ClickHouse/ClickHouse/issues/7156). [#19316](https://github.com/ClickHouse/ClickHouse/pull/19316) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19553](https://github.com/ClickHouse/ClickHouse/issues/19553): Fix system.parts _state column (LOGICAL_ERROR when querying this column, due to incorrect order). [#19346](https://github.com/ClickHouse/ClickHouse/pull/19346) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19469](https://github.com/ClickHouse/ClickHouse/issues/19469): - Fix default value in join types with non-zero default (e.g. some Enums). Closes [#18197](https://github.com/ClickHouse/ClickHouse/issues/18197). [#19360](https://github.com/ClickHouse/ClickHouse/pull/19360) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#19437](https://github.com/ClickHouse/ClickHouse/issues/19437): Fix possible buffer overflow in Uber H3 library. See https://github.com/uber/h3/issues/392. This closes [#19219](https://github.com/ClickHouse/ClickHouse/issues/19219). [#19383](https://github.com/ClickHouse/ClickHouse/pull/19383) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19612](https://github.com/ClickHouse/ClickHouse/issues/19612): Fixed very rare bug that might cause mutation to hang after `DROP/DETACH/REPLACE/MOVE PARTITION`. It was partially fixed by [#15537](https://github.com/ClickHouse/ClickHouse/issues/15537) for the most cases. [#19443](https://github.com/ClickHouse/ClickHouse/pull/19443) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19664](https://github.com/ClickHouse/ClickHouse/issues/19664): Mark distributed batch as broken in case of empty data block in one of files. [#19449](https://github.com/ClickHouse/ClickHouse/pull/19449) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19508](https://github.com/ClickHouse/ClickHouse/issues/19508): Buffer overflow (on memory read) was possible if `addMonth` function was called with specifically crafted arguments. This fixes [#19441](https://github.com/ClickHouse/ClickHouse/issues/19441). This fixes [#19413](https://github.com/ClickHouse/ClickHouse/issues/19413). [#19472](https://github.com/ClickHouse/ClickHouse/pull/19472) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19541](https://github.com/ClickHouse/ClickHouse/issues/19541): Fix SIGSEGV with merge_tree_min_rows_for_concurrent_read/merge_tree_min_bytes_for_concurrent_read=0/UINT64_MAX. [#19528](https://github.com/ClickHouse/ClickHouse/pull/19528) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19642](https://github.com/ClickHouse/ClickHouse/issues/19642): Query CREATE DICTIONARY id expression fix. [#19571](https://github.com/ClickHouse/ClickHouse/pull/19571) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#19638](https://github.com/ClickHouse/ClickHouse/issues/19638): `DROP/DETACH TABLE table ON CLUSTER cluster SYNC` query might hang, it's fixed. Fixes [#19568](https://github.com/ClickHouse/ClickHouse/issues/19568). [#19572](https://github.com/ClickHouse/ClickHouse/pull/19572) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19651](https://github.com/ClickHouse/ClickHouse/issues/19651): Fix use-after-free of the CompressedWriteBuffer in Connection after disconnect. [#19599](https://github.com/ClickHouse/ClickHouse/pull/19599) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19741](https://github.com/ClickHouse/ClickHouse/issues/19741): Fix wrong result of function `neighbor` for `LowCardinality` argument. Fixes [#10333](https://github.com/ClickHouse/ClickHouse/issues/10333). [#19617](https://github.com/ClickHouse/ClickHouse/pull/19617) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19699](https://github.com/ClickHouse/ClickHouse/issues/19699): Some functions with big integers may cause segfault. Big integers is experimental feature. This closes [#19667](https://github.com/ClickHouse/ClickHouse/issues/19667). [#19672](https://github.com/ClickHouse/ClickHouse/pull/19672) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19818](https://github.com/ClickHouse/ClickHouse/issues/19818): Fix a segmentation fault in `bitmapAndnot` function. Fixes [#19668](https://github.com/ClickHouse/ClickHouse/issues/19668). [#19713](https://github.com/ClickHouse/ClickHouse/pull/19713) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#19781](https://github.com/ClickHouse/ClickHouse/issues/19781): Fix crash when nested column name was used in `WHERE` or `PREWHERE`. Fixes [#19755](https://github.com/ClickHouse/ClickHouse/issues/19755). [#19763](https://github.com/ClickHouse/ClickHouse/pull/19763) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19871](https://github.com/ClickHouse/ClickHouse/issues/19871): Fixed stack overflow when using accurate comparison of arithmetic type with string type. [#19773](https://github.com/ClickHouse/ClickHouse/pull/19773) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19811](https://github.com/ClickHouse/ClickHouse/issues/19811): In previous versions, unusual arguments for function arrayEnumerateUniq may cause crash or infinite loop. This closes [#19787](https://github.com/ClickHouse/ClickHouse/issues/19787). [#19788](https://github.com/ClickHouse/ClickHouse/pull/19788) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19941](https://github.com/ClickHouse/ClickHouse/issues/19941): Deadlock was possible if system.text_log is enabled. This fixes [#19874](https://github.com/ClickHouse/ClickHouse/issues/19874). [#19875](https://github.com/ClickHouse/ClickHouse/pull/19875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19937](https://github.com/ClickHouse/ClickHouse/issues/19937): BloomFilter index crash fix. Fixes [#19757](https://github.com/ClickHouse/ClickHouse/issues/19757). [#19884](https://github.com/ClickHouse/ClickHouse/pull/19884) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v20.12.6.29-stable.md b/docs/changelogs/v20.12.6.29-stable.md new file mode 100644 index 00000000000..ac4b27e5ade --- /dev/null +++ b/docs/changelogs/v20.12.6.29-stable.md @@ -0,0 +1,18 @@ +### ClickHouse release v20.12.6.29-stable FIXME as compared to v20.12.5.18-stable + +#### Bug Fix +* Backported in [#19984](https://github.com/ClickHouse/ClickHouse/issues/19984): Background thread which executes `ON CLUSTER` queries might hang waiting for dropped replicated table to do something. It's fixed. [#19684](https://github.com/ClickHouse/ClickHouse/pull/19684) ([yiguolei](https://github.com/yiguolei)). +* Backported in [#20240](https://github.com/ClickHouse/ClickHouse/issues/20240): Fix a bug that moving pieces to destination table may failed in case of launching multiple clickhouse-copiers. [#19743](https://github.com/ClickHouse/ClickHouse/pull/19743) ([madianjun](https://github.com/mdianjun)). +* Backported in [#20080](https://github.com/ClickHouse/ClickHouse/issues/20080): Fix starting the server with tables having default expressions containing dictGet(). Allow getting return type of dictGet() without loading dictionary. [#19805](https://github.com/ClickHouse/ClickHouse/pull/19805) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#20298](https://github.com/ClickHouse/ClickHouse/issues/20298): * Bugfix in StorageJoin. [#20079](https://github.com/ClickHouse/ClickHouse/pull/20079) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#20391](https://github.com/ClickHouse/ClickHouse/issues/20391): The `MongoDB` table engine now establishes connection only when it's going to read data. `ATTACH TABLE` won't try to connect anymore. [#20110](https://github.com/ClickHouse/ClickHouse/pull/20110) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#20328](https://github.com/ClickHouse/ClickHouse/issues/20328): Fix rare server crash on config reload during the shutdown. Fixes [#19689](https://github.com/ClickHouse/ClickHouse/issues/19689). [#20224](https://github.com/ClickHouse/ClickHouse/pull/20224) ([alesapin](https://github.com/alesapin)). +* Backported in [#20332](https://github.com/ClickHouse/ClickHouse/issues/20332): Restrict to `DROP` or `RENAME` version column of `*CollapsingMergeTree` and `ReplacingMergeTree` table engines. [#20300](https://github.com/ClickHouse/ClickHouse/pull/20300) ([alesapin](https://github.com/alesapin)). +* Backported in [#20364](https://github.com/ClickHouse/ClickHouse/issues/20364): Fix too often retries of failed background tasks for `ReplicatedMergeTree` table engines family. This could lead to too verbose logging and increased CPU load. Fixes [#20203](https://github.com/ClickHouse/ClickHouse/issues/20203). [#20335](https://github.com/ClickHouse/ClickHouse/pull/20335) ([alesapin](https://github.com/alesapin)). +* Backported in [#20377](https://github.com/ClickHouse/ClickHouse/issues/20377): Fix null dereference with `join_use_nulls=1`. [#20344](https://github.com/ClickHouse/ClickHouse/pull/20344) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#20360](https://github.com/ClickHouse/ClickHouse/issues/20360): Avoid invalid dereference in RANGE_HASHED() dictionary. [#20345](https://github.com/ClickHouse/ClickHouse/pull/20345) ([Azat Khuzhin](https://github.com/azat)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Backport [#20224](https://github.com/ClickHouse/ClickHouse/issues/20224) to 20.12: Fix access control manager destruction order"'. [#20396](https://github.com/ClickHouse/ClickHouse/pull/20396) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v20.12.7.3-stable.md b/docs/changelogs/v20.12.7.3-stable.md new file mode 100644 index 00000000000..676ab4f7d11 --- /dev/null +++ b/docs/changelogs/v20.12.7.3-stable.md @@ -0,0 +1,9 @@ +### ClickHouse release v20.12.7.3-stable FIXME as compared to v20.12.6.29-stable + +#### Bug Fix +* Backported in [#20683](https://github.com/ClickHouse/ClickHouse/issues/20683): Uninitialized memory read was possible in encrypt/decrypt functions if empty string was passed as IV. This closes [#19391](https://github.com/ClickHouse/ClickHouse/issues/19391). [#19397](https://github.com/ClickHouse/ClickHouse/pull/19397) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#20635](https://github.com/ClickHouse/ClickHouse/issues/20635): Fix rare bug when some replicated operations (like mutation) cannot process some parts after data corruption. Fixes [#19593](https://github.com/ClickHouse/ClickHouse/issues/19593). [#19702](https://github.com/ClickHouse/ClickHouse/pull/19702) ([alesapin](https://github.com/alesapin)). +* Backported in [#20617](https://github.com/ClickHouse/ClickHouse/issues/20617): Check if table function `view` is used in expression list and throw an error. This fixes [#20342](https://github.com/ClickHouse/ClickHouse/issues/20342). [#20350](https://github.com/ClickHouse/ClickHouse/pull/20350) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#20487](https://github.com/ClickHouse/ClickHouse/issues/20487): Fix `LOGICAL_ERROR` for `join_use_nulls=1` when JOIN contains const from SELECT. [#20461](https://github.com/ClickHouse/ClickHouse/pull/20461) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#20614](https://github.com/ClickHouse/ClickHouse/issues/20614): Add proper checks while parsing directory names for async INSERT (fixes SIGSEGV). [#20498](https://github.com/ClickHouse/ClickHouse/pull/20498) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v20.12.8.5-stable.md b/docs/changelogs/v20.12.8.5-stable.md new file mode 100644 index 00000000000..48e20c46928 --- /dev/null +++ b/docs/changelogs/v20.12.8.5-stable.md @@ -0,0 +1,9 @@ +### ClickHouse release v20.12.8.5-stable FIXME as compared to v20.12.7.3-stable + +#### Bug Fix +* Backported in [#20592](https://github.com/ClickHouse/ClickHouse/issues/20592): Fixed inconsistent behavior of dictionary in case of queries where we look for absent keys in dictionary. [#20578](https://github.com/ClickHouse/ClickHouse/pull/20578) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#21048](https://github.com/ClickHouse/ClickHouse/issues/21048): Fix usage of `-Distinct` combinator with `-State` combinator in aggregate functions. [#20866](https://github.com/ClickHouse/ClickHouse/pull/20866) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#21133](https://github.com/ClickHouse/ClickHouse/issues/21133): Fixed behaviour, when `ALTER MODIFY COLUMN` created mutation, that will knowingly fail. [#21007](https://github.com/ClickHouse/ClickHouse/pull/21007) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#21249](https://github.com/ClickHouse/ClickHouse/issues/21249): - Block parallel insertions into storage join. [#21009](https://github.com/ClickHouse/ClickHouse/pull/21009) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#21230](https://github.com/ClickHouse/ClickHouse/issues/21230): Fixes [#21112](https://github.com/ClickHouse/ClickHouse/issues/21112). Fixed bug that could cause duplicates with insert query (if one of the callbacks came a little too late). [#21138](https://github.com/ClickHouse/ClickHouse/pull/21138) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v20.5.1.3833-prestable.md b/docs/changelogs/v20.5.1.3833-prestable.md new file mode 100644 index 00000000000..824fb051914 --- /dev/null +++ b/docs/changelogs/v20.5.1.3833-prestable.md @@ -0,0 +1,380 @@ +### ClickHouse release v20.5.1.3833-prestable FIXME as compared to v20.4.1.3177-prestable + +#### Backward Incompatible Change +* Remove `experimental_use_processors` setting. It is enabled by default. [#10924](https://github.com/ClickHouse/ClickHouse/pull/10924) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* The setting `input_format_with_names_use_header` is enabled by default. It will affect parsing of input formats `-WithNames` and `-WithNamesAndTypes`. [#10937](https://github.com/ClickHouse/ClickHouse/pull/10937) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added a check for the case when user-level setting is specified in a wrong place. User-level settings should be specified in `users.xml` inside `` section for specific user profile (or in `` for default settings). The server won't start with exception message in log. This fixes [#9051](https://github.com/ClickHouse/ClickHouse/issues/9051). If you want to skip the check, you can either move settings to the appropriate place or add `1` to config.xml. [#11449](https://github.com/ClickHouse/ClickHouse/pull/11449) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Return non-Nullable result from COUNT(DISTINCT), and `uniq` aggregate functions family. If all passed values are NULL, return zero instead. This improves SQL compatibility. [#11661](https://github.com/ClickHouse/ClickHouse/pull/11661) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* Support Cassandra as external dictionary. [#4978](https://github.com/ClickHouse/ClickHouse/pull/4978) ([favstovol](https://github.com/favstovol)). +* `SimpleAggregateFunction` now also supports `sumMap`. [#10000](https://github.com/ClickHouse/ClickHouse/pull/10000) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Added OpenCl support and bitonic sort algorithm, which can be used for sorting integer types of data in single column. Needs to be build with flag `-DENABLE_OPENCL=1`. For using bitonic sort algorithm instead of others you need to set `bitonic_sort` for Setting's option `special_sort` and make sure that OpenCL is available. [#10232](https://github.com/ClickHouse/ClickHouse/pull/10232) ([Margarita Konnova [MARK]](https://github.com/margaritiko)). +* Implementation of PostgreSQL wire protocol. [#10242](https://github.com/ClickHouse/ClickHouse/pull/10242) ([Movses Elbakian](https://github.com/MovElb)). +* Selects with final are executed in parallel. Added setting `max_final_threads` to limit the number of threads used. [#10463](https://github.com/ClickHouse/ClickHouse/pull/10463) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Function that extracts from haystack all matching non-overlapping groups with regular expressions, and put those into `Array(Array(String))` column. [#10534](https://github.com/ClickHouse/ClickHouse/pull/10534) ([Vasily Nemkov](https://github.com/Enmk)). +* Added ability to delete a subset of expired rows, which satisfies the condition in WHERE clause. Added ability to replace expired rows with aggregates of them specified in GROUP BY clause. [#10537](https://github.com/ClickHouse/ClickHouse/pull/10537) ([expl0si0nn](https://github.com/expl0si0nn)). +* (Only Linux) Clickhouse server now tries to fallback to ProcfsMetricsProvider when clickhouse binary is not attributed with CAP_NET_ADMIN capability to collect per-query system metrics (for CPU and I/O). [#10544](https://github.com/ClickHouse/ClickHouse/pull/10544) ([Alexander Kazakov](https://github.com/Akazz)). +* - Add Arrow IPC File format (Input and Output) - Fix incorrect work of resetParser() for Parquet Input Format - Add zero-copy optimization for ORC for RandomAccessFiles - Add missing halffloat type for input parquet and ORC formats ... [#10580](https://github.com/ClickHouse/ClickHouse/pull/10580) ([Zhanna](https://github.com/FawnD2)). +* Allowed to profile memory with finer granularity steps than 4 MiB. Added sampling memory profiler to capture random allocations/deallocations. [#10598](https://github.com/ClickHouse/ClickHouse/pull/10598) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add new input format `JSONAsString` that accepts a sequence of JSON objects separated by newlines, spaces and/or commas. [#10607](https://github.com/ClickHouse/ClickHouse/pull/10607) ([Kruglov Pavel](https://github.com/Avogar)). +* Added a new layout ```direct``` which loads all the data directly from the source for each query, without storing or caching data. [#10622](https://github.com/ClickHouse/ClickHouse/pull/10622) ([Artem Streltsov](https://github.com/kekekekule)). +* Default user and database creation on docker image starting. [#10637](https://github.com/ClickHouse/ClickHouse/pull/10637) ([Paramtamtam](https://github.com/tarampampam)). +* Add data type Point (Tuple(Float64, Float64)) and Polygon (Array(Array(Tuple(Float64, Float64))). [#10678](https://github.com/ClickHouse/ClickHouse/pull/10678) ([Alexey Ilyukhov](https://github.com/livace)). +* New function function toStartOfSecond(DateTime64) -> DateTime64 that nullifies sub-second part of DateTime64 value. [#10722](https://github.com/ClickHouse/ClickHouse/pull/10722) ([Vasily Nemkov](https://github.com/Enmk)). +* Added a function `randomString` that generates binary string with random bytes (including zero bytes). [#10733](https://github.com/ClickHouse/ClickHouse/pull/10733) ([Andrei Nekrashevich](https://github.com/axolm)). +* Added `system.licenses` table. This table contains licenses of third-party libraries that are located in `contrib` directory. This closes [#2890](https://github.com/ClickHouse/ClickHouse/issues/2890). [#10795](https://github.com/ClickHouse/ClickHouse/pull/10795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* OFFSET keyword can now be used without an affiliated LIMIT clause. [#10802](https://github.com/ClickHouse/ClickHouse/pull/10802) ([Guillaume Tassery](https://github.com/YiuRULE)). +* Added new complex key direct layout to dictionaries, that does not store anything locally during query execution. [#10850](https://github.com/ClickHouse/ClickHouse/pull/10850) ([Artem Streltsov](https://github.com/kekekekule)). +* Added function `randomFixedString`. [#10866](https://github.com/ClickHouse/ClickHouse/pull/10866) ([Andrei Nekrashevich](https://github.com/axolm)). +* Support `ALTER RENAME COLUMN` for the distributed table engine. Continuation of [#10727](https://github.com/ClickHouse/ClickHouse/issues/10727). Fixes [#10747](https://github.com/ClickHouse/ClickHouse/issues/10747). [#10887](https://github.com/ClickHouse/ClickHouse/pull/10887) ([alesapin](https://github.com/alesapin)). +* Support writes in ODBC Table function [#10554](https://github.com/ClickHouse/ClickHouse/pull/10554) ([ageraab](https://github.com/ageraab)). [#10901](https://github.com/ClickHouse/ClickHouse/pull/10901) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Allow specifying `mongodb://` URI for MongoDB dictionaries. [#10915](https://github.com/ClickHouse/ClickHouse/pull/10915) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Added new functions to import/export DateTime64 as Int64 with various precision: `to-/fromUnixTimestamp64Milli/-Micro/-Nano`. [#10923](https://github.com/ClickHouse/ClickHouse/pull/10923) ([Vasily Nemkov](https://github.com/Enmk)). +* Now support NULL and NOT NULL modifiers for data types in create query. [#11057](https://github.com/ClickHouse/ClickHouse/pull/11057) ([Павел Потемкин](https://github.com/Potya)). +* Add ArrowStream input and output format. [#11088](https://github.com/ClickHouse/ClickHouse/pull/11088) ([hcz](https://github.com/hczhcz)). +* Default S3 credentials and custom auth headers. [#11134](https://github.com/ClickHouse/ClickHouse/pull/11134) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). +* Add query performance metrics based on Linux `perf_events`. [#9545](https://github.com/ClickHouse/ClickHouse/pull/9545) [Andrey Skobtsov](https://github.com/And42). [#11226](https://github.com/ClickHouse/ClickHouse/pull/11226) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Use HTTP client for S3 based on Poco. [#11230](https://github.com/ClickHouse/ClickHouse/pull/11230) ([Pavel Kovalenko](https://github.com/Jokser)). +* Add function `fuzzBits` that randomly flips bits in a string with given probability. [#11237](https://github.com/ClickHouse/ClickHouse/pull/11237) ([Andrei Nekrashevich](https://github.com/axolm)). +* Add `_timestamp_ms` virtual column for Kafka engine (type is `Nullable(DateTime64(3))`). [#11260](https://github.com/ClickHouse/ClickHouse/pull/11260) ([filimonov](https://github.com/filimonov)). +* Add 2 more virtual columns for engine=Kafka to access message headers. [#11283](https://github.com/ClickHouse/ClickHouse/pull/11283) ([filimonov](https://github.com/filimonov)). +* Add `netloc` function for extracting network location, similar to `urlparse(url)`, `netloc` in python. [#11356](https://github.com/ClickHouse/ClickHouse/pull/11356) ([Guillaume Tassery](https://github.com/YiuRULE)). +* Added syntax highligting to clickhouse-client using ReplXX. [#11422](https://github.com/ClickHouse/ClickHouse/pull/11422) ([Tagir Kuskarov](https://github.com/kuskarov)). +* Add SHOW CLUSTER(S) queries. [#11467](https://github.com/ClickHouse/ClickHouse/pull/11467) ([hexiaoting](https://github.com/hexiaoting)). +* Add functions `extractAllGroupsHorizontal(haystack, re)` and `extractAllGroupsVertical(haystack, re)`. [#11554](https://github.com/ClickHouse/ClickHouse/pull/11554) ([Vasily Nemkov](https://github.com/Enmk)). +* Add the `system.asynchronous_metric_log` table that logs historical metrics from `system.asynchronous_metrics`. [#11588](https://github.com/ClickHouse/ClickHouse/pull/11588) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* `minMap` and `maxMap` functions were added. [#11603](https://github.com/ClickHouse/ClickHouse/pull/11603) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Added support for MySQL style global variables syntax (stub). This is needed for compatibility of MySQL protocol. [#11832](https://github.com/ClickHouse/ClickHouse/pull/11832) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Performance Improvement +* Optimization of GROUP BY with respect to table sorting key. [#9113](https://github.com/ClickHouse/ClickHouse/pull/9113) ([Dmitry Rubashkin](https://github.com/dimarub2000)). +* New optimization that takes arithmetic operations out of aggregate functions. [#10047](https://github.com/ClickHouse/ClickHouse/pull/10047) ([Ruslan](https://github.com/kamalov-ruslan)). +* This feature eliminates functions of other keys in GROUP BY section. [#10051](https://github.com/ClickHouse/ClickHouse/pull/10051) ([Victor Grishanin](https://github.com/xPoSx)). +* Add runtime CPU detection to select and dispatch the best function implementation. Add support for codegeneration for multiple targets. This closes [#1017](https://github.com/ClickHouse/ClickHouse/issues/1017). [#10058](https://github.com/ClickHouse/ClickHouse/pull/10058) ([DimasKovas](https://github.com/DimasKovas)). +* Remove duplicate ORDER BY and DISTINCT from subqueries. [#10067](https://github.com/ClickHouse/ClickHouse/pull/10067) ([Mikhail Malafeev](https://github.com/demo-99)). +* Sort bigger parts of the left table in MergeJoin. Buffer left blocks in memory. Add `partial_merge_join_left_table_buffer_bytes` setting to manage the left blocks buffers sizes. [#10601](https://github.com/ClickHouse/ClickHouse/pull/10601) ([Artem Zuikov](https://github.com/4ertus2)). +* Get dictionary and check access rights only once per each call of any function reading external dictionaries. [#10928](https://github.com/ClickHouse/ClickHouse/pull/10928) ([Vitaly Baranov](https://github.com/vitlibar)). +* Improving radix sort by removing some redundant data moves. [#10981](https://github.com/ClickHouse/ClickHouse/pull/10981) ([Arslan Gumerov](https://github.com/g-arslan)). +* Make queries with `sum` aggregate function and without GROUP BY keys to run multiple times faster. [#10992](https://github.com/ClickHouse/ClickHouse/pull/10992) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable mlock of clickhouse binary by default. It will prevent clickhouse executable from being paged out under high IO load. [#11139](https://github.com/ClickHouse/ClickHouse/pull/11139) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improved performance for queries with `ORDER BY` and small `LIMIT` (less, then `max_block_size`). [#11171](https://github.com/ClickHouse/ClickHouse/pull/11171) ([Provet](https://github.com/Provet)). +* Improve performance for INSERT queries via INSERT SELECT or INSERT with clickhouse-client when small blocks are generated (typical case with parallel parsing). This fixes [#11275](https://github.com/ClickHouse/ClickHouse/issues/11275). Fix the issue that CONSTRAINTs were not working for DEFAULT fields. This fixes [#11273](https://github.com/ClickHouse/ClickHouse/issues/11273). Fix the issue that CONSTRAINTS were ignored for TEMPORARY tables. This fixes [#11274](https://github.com/ClickHouse/ClickHouse/issues/11274). [#11276](https://github.com/ClickHouse/ClickHouse/pull/11276) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve performance of `clickhouse-client` in interactive mode when Pretty formats are used. In previous versions, significant amount of time can be spent calculating visible width of UTF-8 string. This closes [#11323](https://github.com/ClickHouse/ClickHouse/issues/11323). [#11323](https://github.com/ClickHouse/ClickHouse/pull/11323) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* New optimization that takes all operations out of "any" function. [#11529](https://github.com/ClickHouse/ClickHouse/pull/11529) ([Ruslan](https://github.com/kamalov-ruslan)). +* Speed up merging in AggregatingMergeTree. This fixes performance regression that was introduced more than a year ago in [#4348](https://github.com/ClickHouse/ClickHouse/issues/4348) (in version 19.3). [#11534](https://github.com/ClickHouse/ClickHouse/pull/11534) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow multiple replicas to assign merges, mutations, partition drop, move and replace concurrently. This closes [#10367](https://github.com/ClickHouse/ClickHouse/issues/10367). [#11639](https://github.com/ClickHouse/ClickHouse/pull/11639) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* This optimization eliminates min/max/any aggregators of GROUP BY keys in SELECT section. [#11667](https://github.com/ClickHouse/ClickHouse/pull/11667) ([Victor Grishanin](https://github.com/xPoSx)). + +#### Improvement +* Allow to pass quota_key in clickhouse-client. This closes [#10227](https://github.com/ClickHouse/ClickHouse/issues/10227). [#10270](https://github.com/ClickHouse/ClickHouse/pull/10270) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added system tables for users, roles, grants, settings profiles, quotas, row policies; added commands SHOW USER, SHOW [CURRENT|ENABLED] ROLES, SHOW SETTINGS PROFILES. [#10387](https://github.com/ClickHouse/ClickHouse/pull/10387) ([Vitaly Baranov](https://github.com/vitlibar)). +* - Adding support for `INSERT INTO [db.]table WATCH` query. [#10498](https://github.com/ClickHouse/ClickHouse/pull/10498) ([vzakaznikov](https://github.com/vzakaznikov)). +* Possibility to work with S3 through proxies. [#10576](https://github.com/ClickHouse/ClickHouse/pull/10576) ([Pavel Kovalenko](https://github.com/Jokser)). +* Added `move_ttl_info` to `system.parts` in order to provide introspection of move TTL functionality. [#10591](https://github.com/ClickHouse/ClickHouse/pull/10591) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Make pointInPolygon work with non-constant polygon. PointInPolygon now can take Array(Array(Tuple(..., ...))) as second argument, array of polygon and holes. [#10623](https://github.com/ClickHouse/ClickHouse/pull/10623) ([Alexey Ilyukhov](https://github.com/livace)). +* Print a message if clickhouse-client is newer than clickhouse-server. [#10627](https://github.com/ClickHouse/ClickHouse/pull/10627) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change HTTP response code in case of some parse errors to 400 Bad Request. This fix [#10636](https://github.com/ClickHouse/ClickHouse/issues/10636). [#10640](https://github.com/ClickHouse/ClickHouse/pull/10640) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added a check for meaningless codecs and a setting `allow_suspicious_codecs` to control this check. This closes [#4966](https://github.com/ClickHouse/ClickHouse/issues/4966). [#10645](https://github.com/ClickHouse/ClickHouse/pull/10645) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adding support for ALTER RENAME COLUMN query to Distributed table engine. [#10727](https://github.com/ClickHouse/ClickHouse/pull/10727) ([vzakaznikov](https://github.com/vzakaznikov)). +* Possibility to configure proxy-resolver for DiskS3. [#10744](https://github.com/ClickHouse/ClickHouse/pull/10744) ([Pavel Kovalenko](https://github.com/Jokser)). +* Better DNS exception message. This fixes [#10813](https://github.com/ClickHouse/ClickHouse/issues/10813). [#10828](https://github.com/ClickHouse/ClickHouse/pull/10828) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Ensure that `varSamp`, `varPop` cannot return negative results due to numerical errors and that `stddevSamp`, `stddevPop` cannot be calculated from negative variance. This fixes [#10532](https://github.com/ClickHouse/ClickHouse/issues/10532). [#10829](https://github.com/ClickHouse/ClickHouse/pull/10829) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Get rid of exception from replicated queue during server shutdown. Fixes [#10819](https://github.com/ClickHouse/ClickHouse/issues/10819). [#10841](https://github.com/ClickHouse/ClickHouse/pull/10841) ([alesapin](https://github.com/alesapin)). +* The `clickhouse-format` tool is now able to format multiple queries when the `-n` argument is used. [#10852](https://github.com/ClickHouse/ClickHouse/pull/10852) ([Darío](https://github.com/dgrr)). +* Provide synonims for some data types. [#10856](https://github.com/ClickHouse/ClickHouse/pull/10856) ([Павел Потемкин](https://github.com/Potya)). +* Introduce `min_insert_block_size_rows_for_materialized_views ` , `min_insert_block_size_bytes_for_materialized_views` settings. This settings are similar to `min_insert_block_size_rows` and `min_insert_block_size_bytes`, but applied only for blocks inserted into `MATERIALIZED VIEW`. It helps to control blocks squashing while pushing to MVs and avoid excessive memory usage. [#10858](https://github.com/ClickHouse/ClickHouse/pull/10858) ([Azat Khuzhin](https://github.com/azat)). +* Respect prefer_localhost_replica/load_balancing on INSERT into Distributed. [#10867](https://github.com/ClickHouse/ClickHouse/pull/10867) ([Azat Khuzhin](https://github.com/azat)). +* Allow large UInt types as the index in function `tupleElement`. [#10874](https://github.com/ClickHouse/ClickHouse/pull/10874) ([hcz](https://github.com/hczhcz)). +* Support for unicode whitespaces in queries. This helps when queries are copy-pasted from Word or from web page. This fixes [#10896](https://github.com/ClickHouse/ClickHouse/issues/10896). [#10903](https://github.com/ClickHouse/ClickHouse/pull/10903) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Some additions and cleanup for [#10232](https://github.com/ClickHouse/ClickHouse/issues/10232). [#10934](https://github.com/ClickHouse/ClickHouse/pull/10934) ([Artem Zuikov](https://github.com/4ertus2)). +* Set thread names for internal threads of rdkafka library. Make logs from rdkafka available in server logs. [#10983](https://github.com/ClickHouse/ClickHouse/pull/10983) ([Azat Khuzhin](https://github.com/azat)). +* Remove data on explicit `DROP DATABASE` for `Memory` database engine. Fixes [#10557](https://github.com/ClickHouse/ClickHouse/issues/10557). [#11021](https://github.com/ClickHouse/ClickHouse/pull/11021) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add `NCHAR` and `NVARCHAR` synonims for data types. [#11025](https://github.com/ClickHouse/ClickHouse/pull/11025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Apply `TTL` for old data, after `ALTER MODIFY TTL` query. This behaviour is controlled by setting `materialize_ttl_after_modify`, which is enabled by default. [#11042](https://github.com/ClickHouse/ClickHouse/pull/11042) ([Anton Popov](https://github.com/CurtizJ)). +* Now `dictGet*` functions accept table names. [#11050](https://github.com/ClickHouse/ClickHouse/pull/11050) ([Vitaly Baranov](https://github.com/vitlibar)). +* Show authentication type in table system.users and while executing SHOW CREATE USER query. [#11080](https://github.com/ClickHouse/ClickHouse/pull/11080) ([Vitaly Baranov](https://github.com/vitlibar)). +* Enable percpu_arena:percpu for jemalloc (This will reduce memory fragmentation due to thread pool). [#11084](https://github.com/ClickHouse/ClickHouse/pull/11084) ([Azat Khuzhin](https://github.com/azat)). +* Add port() function (to extract port from URL). [#11120](https://github.com/ClickHouse/ClickHouse/pull/11120) ([Azat Khuzhin](https://github.com/azat)). +* Resolved [#7224](https://github.com/ClickHouse/ClickHouse/issues/7224): added `FailedQuery`, `FailedSelectQuery` and `FailedInsertQuery` metrics to `system.events` table. [#11151](https://github.com/ClickHouse/ClickHouse/pull/11151) ([Nikita Orlov](https://github.com/naorlov)). +* The query log is now enabled by default. [#11184](https://github.com/ClickHouse/ClickHouse/pull/11184) ([Ivan Blinkov](https://github.com/blinkov)). +* When parsing C-style backslash escapes in string literals, VALUES and various text formats (this is an extension to SQL standard that is endemic for ClickHouse and MySQL), keep backslash if unknown escape sequence is found (e.g. `\%` or `\w`) that will make usage of `LIKE` and `match` regular expressions more convenient (it's enough to write `name LIKE 'used\_cars'` instead of `name LIKE 'used\\_cars'`) and more compatible at the same time. This fixes [#10922](https://github.com/ClickHouse/ClickHouse/issues/10922). [#11208](https://github.com/ClickHouse/ClickHouse/pull/11208) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add support for multi-word data type names (such as `DOUBLE PRECISION` and `CHAR VARYING`) for better SQL compatibility. [#11214](https://github.com/ClickHouse/ClickHouse/pull/11214) ([Павел Потемкин](https://github.com/Potya)). +* Keep the value of `DistributedFilesToInsert` metric on exceptions. In previous versions, the value was set when we are going to send some files, but it is zero, if there was an exception and some files are still pending. Now it corresponds to the number of pending files in filesystem. [#11220](https://github.com/ClickHouse/ClickHouse/pull/11220) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support kafka_client_id parameter for Kafka tables. It also changes the default `client.id` used by ClickHouse when communicating with Kafka to be more verbose and usable. [#11252](https://github.com/ClickHouse/ClickHouse/pull/11252) ([filimonov](https://github.com/filimonov)). +* Update librdkafka to version [1.4.2](https://github.com/edenhill/librdkafka/releases/tag/v1.4.2). [#11256](https://github.com/ClickHouse/ClickHouse/pull/11256) ([filimonov](https://github.com/filimonov)). +* Support (U)Int8, (U)Int16, Date in ASOF JOIN. [#11301](https://github.com/ClickHouse/ClickHouse/pull/11301) ([Artem Zuikov](https://github.com/4ertus2)). +* Better exception message in case when there is shortage of memory mappings. This closes [#11027](https://github.com/ClickHouse/ClickHouse/issues/11027). [#11316](https://github.com/ClickHouse/ClickHouse/pull/11316) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add setting "output_format_pretty_max_value_width". If value is longer, it will be cut to avoid output of too large values in terminal. This closes [#11140](https://github.com/ClickHouse/ClickHouse/issues/11140). [#11324](https://github.com/ClickHouse/ClickHouse/pull/11324) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trailing whitespaces from formatted queries in `clickhouse-client` or `clickhouse-format` in some cases. [#11325](https://github.com/ClickHouse/ClickHouse/pull/11325) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better log messages in while reloading configuration. [#11341](https://github.com/ClickHouse/ClickHouse/pull/11341) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Suppress output of cancelled queries in clickhouse-client. In previous versions result may continue to print in terminal even after you press Ctrl+C to cancel query. This closes [#9473](https://github.com/ClickHouse/ClickHouse/issues/9473). [#11342](https://github.com/ClickHouse/ClickHouse/pull/11342) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* OPTIMIZE FINAL will force merge even if concurrent merges are performed. This closes [#11309](https://github.com/ClickHouse/ClickHouse/issues/11309) and closes [#11322](https://github.com/ClickHouse/ClickHouse/issues/11322). [#11346](https://github.com/ClickHouse/ClickHouse/pull/11346) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support for all format settings in Kafka, expose some setting on table level, adjust the defaults for better performance. [#11388](https://github.com/ClickHouse/ClickHouse/pull/11388) ([filimonov](https://github.com/filimonov)). +* Add system.distribution_queue table. [#11394](https://github.com/ClickHouse/ClickHouse/pull/11394) ([Azat Khuzhin](https://github.com/azat)). +* ON CLUSTER support for SYSTEM {FLUSH DISTRIBUTED,STOP/START DISTRIBUTED SEND}. [#11415](https://github.com/ClickHouse/ClickHouse/pull/11415) ([Azat Khuzhin](https://github.com/azat)). +* Now history file is updated after each query and there is no race condition if multiple clients use one history file. This fixes [#9897](https://github.com/ClickHouse/ClickHouse/issues/9897). [#11453](https://github.com/ClickHouse/ClickHouse/pull/11453) ([Tagir Kuskarov](https://github.com/kuskarov)). +* Automatically update DNS cache, which is used to check if user is allowed to connect from an address. [#11487](https://github.com/ClickHouse/ClickHouse/pull/11487) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Adding support for PREWHERE in live view tables. [#11495](https://github.com/ClickHouse/ClickHouse/pull/11495) ([vzakaznikov](https://github.com/vzakaznikov)). +* Improve `enable_optimize_predicate_expression=1` logic for VIEW. [#11513](https://github.com/ClickHouse/ClickHouse/pull/11513) ([Artem Zuikov](https://github.com/4ertus2)). +* Better exception message when cannot parse columns declaration list. This closes [#10403](https://github.com/ClickHouse/ClickHouse/issues/10403). [#11537](https://github.com/ClickHouse/ClickHouse/pull/11537) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Optimize memory usage when reading a response from an S3 HTTP client. [#11561](https://github.com/ClickHouse/ClickHouse/pull/11561) ([Pavel Kovalenko](https://github.com/Jokser)). +* Improve `multiple_joins_rewriter_version=2` logic. Fix unknown columns error for lambda aliases. [#11587](https://github.com/ClickHouse/ClickHouse/pull/11587) ([Artem Zuikov](https://github.com/4ertus2)). +* Make more input format work with Kafka engine. Fix the issue with premature flushes. Fix the performance issue when `kafka_num_consumers` is greater than number of partitions in topic. [#11599](https://github.com/ClickHouse/ClickHouse/pull/11599) ([filimonov](https://github.com/filimonov)). +* https://github.com/ClickHouse/ClickHouse/pull/7572#issuecomment-642815377 Support config default HTTPHandlers. [#11628](https://github.com/ClickHouse/ClickHouse/pull/11628) ([Winter Zhang](https://github.com/zhang2014)). +* Add round_robin load_balancing. [#11645](https://github.com/ClickHouse/ClickHouse/pull/11645) ([Azat Khuzhin](https://github.com/azat)). +* Allow comparison of numbers with constant string in comparison operators, IN and VALUES sections. [#11647](https://github.com/ClickHouse/ClickHouse/pull/11647) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow comparison with constant strings by implicit conversions when analysing index conditions on other types. This may close [#11630](https://github.com/ClickHouse/ClickHouse/issues/11630). [#11648](https://github.com/ClickHouse/ClickHouse/pull/11648) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow using `groupArrayArray` and `groupUniqArrayArray` as `SimpleAggregateFunction`. [#11650](https://github.com/ClickHouse/ClickHouse/pull/11650) ([Volodymyr Kuznetsov](https://github.com/ksvladimir)). +* Skip empty parameters in requested URL. They may appear when you write `http://localhost:8123/?&a=b` or `http://localhost:8123/?a=b&&c=d`. This closes [#10749](https://github.com/ClickHouse/ClickHouse/issues/10749). [#11651](https://github.com/ClickHouse/ClickHouse/pull/11651) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to DROP replicated table if the metadata in ZooKeeper was already removed and does not exist (this is also the case when using TestKeeper for testing and the server was restarted). Allow to RENAME replicated table even if there is an error communicating with ZooKeeper. This fixes [#10720](https://github.com/ClickHouse/ClickHouse/issues/10720). [#11652](https://github.com/ClickHouse/ClickHouse/pull/11652) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Return NULL/zero when value is not parsed completely in parseDateTimeBestEffortOrNull/Zero functions. This fixes [#7876](https://github.com/ClickHouse/ClickHouse/issues/7876). [#11653](https://github.com/ClickHouse/ClickHouse/pull/11653) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added column `position` to `system.columns` table and `column_position` to `system.parts_columns` table. It contains ordinal position of a column in a table starting with 1. This closes [#7744](https://github.com/ClickHouse/ClickHouse/issues/7744). [#11655](https://github.com/ClickHouse/ClickHouse/pull/11655) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't use debug info from ELF file if it doesn't correspond to the running binary. It is needed to avoid printing wrong function names and source locations in stack traces. This fixes [#7514](https://github.com/ClickHouse/ClickHouse/issues/7514). [#11657](https://github.com/ClickHouse/ClickHouse/pull/11657) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Clear password from command line in `clickhouse-client` and `clickhouse-benchmark` if the user has specified it with explicit value. This prevents password exposure by `ps` and similar tools. [#11665](https://github.com/ClickHouse/ClickHouse/pull/11665) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Multiple names are now allowed in commands: CREATE USER, CREATE ROLE, ALTER USER, SHOW CREATE USER, SHOW GRANTS and so on. [#11670](https://github.com/ClickHouse/ClickHouse/pull/11670) ([Vitaly Baranov](https://github.com/vitlibar)). +* When multiline query is printed to server log, the lines are joined. Make it to work correct in case of multiline string literals, identifiers and single-line comments. This fixes [#3853](https://github.com/ClickHouse/ClickHouse/issues/3853). [#11686](https://github.com/ClickHouse/ClickHouse/pull/11686) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Emit warning instead of error in server log at startup if we cannot listen one of the listen addresses (e.g. IPv6 is unavailable inside Docker). Note that if server fails to listen all listed addresses, it will refuse to startup as before. This fixes [#4406](https://github.com/ClickHouse/ClickHouse/issues/4406). [#11687](https://github.com/ClickHouse/ClickHouse/pull/11687) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added support for distributed `DDL` (update/delete/drop partition) on cross replication clusters. [#11703](https://github.com/ClickHouse/ClickHouse/pull/11703) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add `cast_keep_nullable` setting. If set `CAST(something_nullable AS Type)` return `Nullable(Type)`. [#11733](https://github.com/ClickHouse/ClickHouse/pull/11733) ([Artem Zuikov](https://github.com/4ertus2)). +* Add more `jemalloc` statistics to `system.asynchronous_metrics`, and ensure that we see up-to-date values for them. [#11748](https://github.com/ClickHouse/ClickHouse/pull/11748) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add ability to set `MATERIALIZED` default type for primary key columns and columns with secondary indices. [#11786](https://github.com/ClickHouse/ClickHouse/pull/11786) ([alesapin](https://github.com/alesapin)). +* Remove leader election, step 3: remove yielding of leadership; remove sending queries to leader. [#11795](https://github.com/ClickHouse/ClickHouse/pull/11795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added `hostname` as an alias to function `hostName`. This feature was suggested by Victor Tarnavskiy from Yandex.Metrica. [#11821](https://github.com/ClickHouse/ClickHouse/pull/11821) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix sleep invocation in signal handler. It was sleeping for less amount of time than expected. [#11825](https://github.com/ClickHouse/ClickHouse/pull/11825) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Slightly improve diagnostic of reading decimal from string. This closes [#10202](https://github.com/ClickHouse/ClickHouse/issues/10202). [#11829](https://github.com/ClickHouse/ClickHouse/pull/11829) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* When reading Decimal value, cut extra digits after point. This behaviour is more compatible with MySQL and PostgreSQL. This fixes [#10202](https://github.com/ClickHouse/ClickHouse/issues/10202). [#11831](https://github.com/ClickHouse/ClickHouse/pull/11831) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Fixed error with "max_rows_to_sort" limit. [#10268](https://github.com/ClickHouse/ClickHouse/pull/10268) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixes: [#10263](https://github.com/ClickHouse/ClickHouse/issues/10263) (after that PR dist send via INSERT had been postponing on each INSERT) Fixes: [#8756](https://github.com/ClickHouse/ClickHouse/issues/8756) (that PR breaks distributed sends with all of the following conditions met (unlikely setup for now I guess): `internal_replication == false`, multiple local shards (activates the hardlinking code) and `distributed_storage_policy` (makes `link(2)` fails on `EXDEV`)). [#10486](https://github.com/ClickHouse/ClickHouse/pull/10486) ([Azat Khuzhin](https://github.com/azat)). +* Disable GROUP BY sharding_key optimization by default (`optimize_distributed_group_by_sharding_key` had been introduced and turned of by default, due to trickery of sharding_key analyzing, simple example is `if` in sharding key) and fix it for WITH ROLLUP/CUBE/TOTALS. [#10516](https://github.com/ClickHouse/ClickHouse/pull/10516) ([Azat Khuzhin](https://github.com/azat)). +* Fix index corruption, which may accur in some cases after merge compact parts into another compact part. [#10531](https://github.com/ClickHouse/ClickHouse/pull/10531) ([Anton Popov](https://github.com/CurtizJ)). +* Implemented comparison between DateTime64 and String values (just like for DateTime). [#10560](https://github.com/ClickHouse/ClickHouse/pull/10560) ([Vasily Nemkov](https://github.com/Enmk)). +* Fix SELECT of column ALIAS which default expression type different from column type. [#10563](https://github.com/ClickHouse/ClickHouse/pull/10563) ([Azat Khuzhin](https://github.com/azat)). +* Fix error `the BloomFilter false positive must be a double number between 0 and 1` [#10551](https://github.com/ClickHouse/ClickHouse/issues/10551). [#10569](https://github.com/ClickHouse/ClickHouse/pull/10569) ([Winter Zhang](https://github.com/zhang2014)). +* This PR fixes possible crash when `createDictionary()` is called before `loadStoredObject()` has finished. [#10587](https://github.com/ClickHouse/ClickHouse/pull/10587) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed handling condition variable for synchronous mutations. In some cases signals to that condition variable could be lost. [#10588](https://github.com/ClickHouse/ClickHouse/pull/10588) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fixed incorrect scalar results inside inner query of `MATERIALIZED VIEW` in case if this query contained dependent table. [#10603](https://github.com/ClickHouse/ClickHouse/pull/10603) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* On `SYSTEM DROP DNS CACHE` query also drop caches, which are used to check if user is allowed to connect from some IP addresses. [#10608](https://github.com/ClickHouse/ClickHouse/pull/10608) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix segfault in StorageBuffer when exception on server startup. Fixes [#10550](https://github.com/ClickHouse/ClickHouse/issues/10550). [#10609](https://github.com/ClickHouse/ClickHouse/pull/10609) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix optimize_skip_unused_shards with LowCardinality. [#10611](https://github.com/ClickHouse/ClickHouse/pull/10611) ([Azat Khuzhin](https://github.com/azat)). +* Fix predicates optimization for distributed queries (`enable_optimize_predicate_expression=1`) for queries with `HAVING` section (i.e. when filtering on the server initiator is required), by preserving the order of expressions (and this is enough to fix), and also force aggregator use column names over indexes. Fixes: [#10613](https://github.com/ClickHouse/ClickHouse/issues/10613), [#11413](https://github.com/ClickHouse/ClickHouse/issues/11413). [#10621](https://github.com/ClickHouse/ClickHouse/pull/10621) ([Azat Khuzhin](https://github.com/azat)). +* Fix nullptr dereference in StorageBuffer if server was shutdown before table startup. [#10641](https://github.com/ClickHouse/ClickHouse/pull/10641) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bug which locks concurrent alters when table has a lot of parts. [#10659](https://github.com/ClickHouse/ClickHouse/pull/10659) ([alesapin](https://github.com/alesapin)). +* Fix possible incorrect number of rows for queries with `LIMIT`. Fixes [#10566](https://github.com/ClickHouse/ClickHouse/issues/10566), [#10709](https://github.com/ClickHouse/ClickHouse/issues/10709). [#10660](https://github.com/ClickHouse/ClickHouse/pull/10660) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix the lack of parallel execution of remote queries with `distributed_aggregation_memory_efficient` enabled. Fixes [#10655](https://github.com/ClickHouse/ClickHouse/issues/10655). [#10664](https://github.com/ClickHouse/ClickHouse/pull/10664) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix order of parameters in AggregateTransform constructor. [#10667](https://github.com/ClickHouse/ClickHouse/pull/10667) ([palasonic1](https://github.com/palasonic1)). +* Fixed bug, which causes http requests stuck on client close when `readonly=2` and `cancel_http_readonly_queries_on_client_close=1`. Fixes [#7939](https://github.com/ClickHouse/ClickHouse/issues/7939), [#7019](https://github.com/ClickHouse/ClickHouse/issues/7019), [#7736](https://github.com/ClickHouse/ClickHouse/issues/7736), [#7091](https://github.com/ClickHouse/ClickHouse/issues/7091). [#10684](https://github.com/ClickHouse/ClickHouse/pull/10684) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix multiple usages of `IN` operator with the identical set in one query. [#10686](https://github.com/ClickHouse/ClickHouse/pull/10686) ([Anton Popov](https://github.com/CurtizJ)). +* Fix atomicity of HTTP insert. This fixes [#9666](https://github.com/ClickHouse/ClickHouse/issues/9666). [#10687](https://github.com/ClickHouse/ClickHouse/pull/10687) ([Andrew Onyshchuk](https://github.com/oandrew)). +* Fix disappearing totals. Totals could have being filtered if query had had join or subquery with external where condition. Fixes [#10674](https://github.com/ClickHouse/ClickHouse/issues/10674). [#10698](https://github.com/ClickHouse/ClickHouse/pull/10698) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible buffer overflow in function `h3EdgeAngle`. [#10711](https://github.com/ClickHouse/ClickHouse/pull/10711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix usage of primary key wrapped into a function with 'FINAL' modifier and 'ORDER BY' optimization. [#10715](https://github.com/ClickHouse/ClickHouse/pull/10715) ([Anton Popov](https://github.com/CurtizJ)). +* Fix data corruption for `LowCardinality(FixedString)` key column in `SummingMergeTree` which could have happened after merge. Fixes [#10489](https://github.com/ClickHouse/ClickHouse/issues/10489). [#10721](https://github.com/ClickHouse/ClickHouse/pull/10721) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in `generateRandom` with nested types. Fixes [#10583](https://github.com/ClickHouse/ClickHouse/issues/10583). [#10734](https://github.com/ClickHouse/ClickHouse/pull/10734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix combinator -OrNull and -OrDefault when combined with -State. [#10741](https://github.com/ClickHouse/ClickHouse/pull/10741) ([hcz](https://github.com/hczhcz)). +* Fix `parallel_view_processing` behavior. Now all insertions into `MATERIALIZED VIEW` without exception should be finished if exception happened. Fixes [#10241](https://github.com/ClickHouse/ClickHouse/issues/10241). [#10757](https://github.com/ClickHouse/ClickHouse/pull/10757) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix avgWeighted when using floating-point weight over multiple shards. [#10758](https://github.com/ClickHouse/ClickHouse/pull/10758) ([Baudouin Giard](https://github.com/bgiard)). +* Get rid of old libunwind patches. https://github.com/ClickHouse-Extras/libunwind/commit/500aa227911bd185a94bfc071d68f4d3b03cb3b1#r39048012 This allows to disable `-fno-omit-frame-pointer` in `clang` builds that improves performance at least by 1% in average. [#10761](https://github.com/ClickHouse/ClickHouse/pull/10761) ([Amos Bird](https://github.com/amosbird)). +* Make use of `src_type` for correct type conversion in key conditions. Fixes [#6287](https://github.com/ClickHouse/ClickHouse/issues/6287). [#10791](https://github.com/ClickHouse/ClickHouse/pull/10791) ([Andrew Onyshchuk](https://github.com/oandrew)). +* Fix UBSan and MSan report in DateLUT. [#10798](https://github.com/ClickHouse/ClickHouse/pull/10798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix the issue with ODBC bridge when no quoting of identifiers is requested. This fixes [#7984](https://github.com/ClickHouse/ClickHouse/issues/7984). [#10821](https://github.com/ClickHouse/ClickHouse/pull/10821) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix columns order after Block::sortColumns() (also add a test that shows that it affects some real use case - Buffer engine). [#10826](https://github.com/ClickHouse/ClickHouse/pull/10826) ([Azat Khuzhin](https://github.com/azat)). +* Fix potential read of uninitialized memory in cache dictionary. [#10834](https://github.com/ClickHouse/ClickHouse/pull/10834) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now constraints are updated if the column participating in `CONSTRAINT` expression was renamed. Fixes [#10844](https://github.com/ClickHouse/ClickHouse/issues/10844). [#10847](https://github.com/ClickHouse/ClickHouse/pull/10847) ([alesapin](https://github.com/alesapin)). +* Fixed bug in `ReplicatedMergeTree` which might cause some `ALTER` on `OPTIMIZE` query to hang waiting for some replica after it become inactive. [#10849](https://github.com/ClickHouse/ClickHouse/pull/10849) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed `WATCH` hangs after `LiveView` table was dropped from database with `Atomic` engine. [#10859](https://github.com/ClickHouse/ClickHouse/pull/10859) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix SIGSEGV in StringHashTable (if such key does not exist). [#10870](https://github.com/ClickHouse/ClickHouse/pull/10870) ([Azat Khuzhin](https://github.com/azat)). +* Fix backward compatibility with tuples in Distributed tables. [#10889](https://github.com/ClickHouse/ClickHouse/pull/10889) ([Anton Popov](https://github.com/CurtizJ)). +* Fix possible race which could happen when you get result from aggregate function state from multiple thread for the same column. The only way (which I found) it can happen is when you use `finalizeAggregation` function while reading from table with `Memory` engine which stores `AggregateFunction` state for `quanite*` function. [#10890](https://github.com/ClickHouse/ClickHouse/pull/10890) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Now it's possible to execute multiple `ALTER RENAME` like `a TO b, c TO a`. [#10895](https://github.com/ClickHouse/ClickHouse/pull/10895) ([alesapin](https://github.com/alesapin)). +* Fix for the hang which was happening sometimes during DROP of table engine=Kafka (or during server restarts). [#10910](https://github.com/ClickHouse/ClickHouse/pull/10910) ([filimonov](https://github.com/filimonov)). +* Fix crash in `SELECT count(notNullIn(NULL, []))`. [#10920](https://github.com/ClickHouse/ClickHouse/pull/10920) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Avoid sending partially written files by the DistributedBlockOutputStream. [#10940](https://github.com/ClickHouse/ClickHouse/pull/10940) ([Azat Khuzhin](https://github.com/azat)). +* Fix incompatibility of two-level aggregation between versions 20.1 and earlier. This incompatibility happens when different versions of ClickHouse are used on initiator node and remote nodes and the size of GROUP BY result is large and aggregation is performed by a single String field. It leads to several unmerged rows for a single key in result. [#10952](https://github.com/ClickHouse/ClickHouse/pull/10952) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix incorrect raw data size in method getRawData(). [#10964](https://github.com/ClickHouse/ClickHouse/pull/10964) ([Igr](https://github.com/ObjatieGroba)). +* Fix server crash on concurrent `ALTER` and `DROP DATABASE` queries with `Atomic` database engine. [#10968](https://github.com/ClickHouse/ClickHouse/pull/10968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix metadata (relative path for rename) and data (relative path for symlink) handling for Atomic database. [#10980](https://github.com/ClickHouse/ClickHouse/pull/10980) ([Azat Khuzhin](https://github.com/azat)). +* Fix very rare potential use-after-free error in MergeTree if table was not created successfully. [#10986](https://github.com/ClickHouse/ClickHouse/pull/10986) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix memory tracking for two-level GROUP BY when not all rows read from Aggregator (TCP). [#11022](https://github.com/ClickHouse/ClickHouse/pull/11022) ([Azat Khuzhin](https://github.com/azat)). +* Fixed parsing of S3 URLs. [#11036](https://github.com/ClickHouse/ClickHouse/pull/11036) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Now it's possible to `ADD/DROP` and `RENAME` the same one column in a single `ALTER` query. Exception message for simultaneous `MODIFY` and `RENAME` became more clear. Partially fixes [#10669](https://github.com/ClickHouse/ClickHouse/issues/10669). [#11037](https://github.com/ClickHouse/ClickHouse/pull/11037) ([alesapin](https://github.com/alesapin)). +* Fixed parseDateTime64BestEffort argument resolution bugs. [#10925](https://github.com/ClickHouse/ClickHouse/issues/10925). [#11038](https://github.com/ClickHouse/ClickHouse/pull/11038) ([Vasily Nemkov](https://github.com/Enmk)). +* Fixes the potential missed data during termination of Kafka engine table. [#11048](https://github.com/ClickHouse/ClickHouse/pull/11048) ([filimonov](https://github.com/filimonov)). +* Fix error `No such name in Block::erase()` when JOIN appears with PREWHERE or `optimize_move_to_prewhere` makes PREWHERE from WHERE. [#11051](https://github.com/ClickHouse/ClickHouse/pull/11051) ([Artem Zuikov](https://github.com/4ertus2)). +* Fixed memory leak in registerDiskS3. [#11074](https://github.com/ClickHouse/ClickHouse/pull/11074) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fixed deadlock during server startup after update with changes in structure of system log tables. [#11106](https://github.com/ClickHouse/ClickHouse/pull/11106) ([alesapin](https://github.com/alesapin)). +* Remove logging from mutation finalization task if nothing was finalized. [#11109](https://github.com/ClickHouse/ClickHouse/pull/11109) ([alesapin](https://github.com/alesapin)). +* Fix excessive reserving of threads for simple queries (optimization for reducing the number of threads, which was partly broken after changes in pipeline). [#11114](https://github.com/ClickHouse/ClickHouse/pull/11114) ([Azat Khuzhin](https://github.com/azat)). +* Fix for the hang which was happening sometimes during DROP of table engine=Kafka (or during server restarts). [#11145](https://github.com/ClickHouse/ClickHouse/pull/11145) ([filimonov](https://github.com/filimonov)). +* Fix Kafka performance issue related to reschedules based on limits, which were always applied. [#11149](https://github.com/ClickHouse/ClickHouse/pull/11149) ([filimonov](https://github.com/filimonov)). +* If data skipping index is dependent on columns that are going to be modified during background merge (for SummingMergeTree, AggregatingMergeTree as well as for TTL GROUP BY), it was calculated incorrectly. This issue is fixed by moving index calculation after merge so the index is calculated on merged data. [#11162](https://github.com/ClickHouse/ClickHouse/pull/11162) ([Azat Khuzhin](https://github.com/azat)). +* Fixed S3 globbing which could fail in case of more than 1000 keys and some backends. [#11179](https://github.com/ClickHouse/ClickHouse/pull/11179) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix possible error `Cannot capture column` for higher-order functions with `Array(Array(LowCardinality))` captured argument. [#11185](https://github.com/ClickHouse/ClickHouse/pull/11185) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Now `primary.idx` will be checked if it's defined in `CREATE` query. [#11199](https://github.com/ClickHouse/ClickHouse/pull/11199) ([alesapin](https://github.com/alesapin)). +* Fix possible exception `Invalid status for associated output`. [#11200](https://github.com/ClickHouse/ClickHouse/pull/11200) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix error `Block structure mismatch in QueryPipeline` while reading from `VIEW` with constants in inner query. Fixes [#11181](https://github.com/ClickHouse/ClickHouse/issues/11181). [#11205](https://github.com/ClickHouse/ClickHouse/pull/11205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed a bug when cache-dictionary could return default value instead of normal (when there are only expired keys). This affects only string fields. [#11233](https://github.com/ClickHouse/ClickHouse/pull/11233) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix crash while reading malformed data in Protobuf format. This fixes [#5957](https://github.com/ClickHouse/ClickHouse/issues/5957), fixes [#11203](https://github.com/ClickHouse/ClickHouse/issues/11203). [#11258](https://github.com/ClickHouse/ClickHouse/pull/11258) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix wrong markup in documentation. [#11263](https://github.com/ClickHouse/ClickHouse/pull/11263) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix crash when SET DEFAULT ROLE is called with wrong arguments. This fixes [#10586](https://github.com/ClickHouse/ClickHouse/issues/10586). [#11278](https://github.com/ClickHouse/ClickHouse/pull/11278) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix bug when query speed estimation can be incorrect and the limit of `min_execution_speed` may not work or work incorrectly if the query is throttled by `max_network_bandwidth`, `max_execution_speed` or `priority` settings. Change the default value of `timeout_before_checking_execution_speed` to non-zero, because otherwise the settings `min_execution_speed` and `max_execution_speed` have no effect. This fixes [#11297](https://github.com/ClickHouse/ClickHouse/issues/11297). This fixes [#5732](https://github.com/ClickHouse/ClickHouse/issues/5732). This fixes [#6228](https://github.com/ClickHouse/ClickHouse/issues/6228). Usability improvement: avoid concatenation of exception message with progress bar in `clickhouse-client`. [#11296](https://github.com/ClickHouse/ClickHouse/pull/11296) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix the issue when index analysis cannot work if a table has Array column in primary key and if a query is filtering by this column with `empty` or `notEmpty` functions. This fixes [#11286](https://github.com/ClickHouse/ClickHouse/issues/11286). [#11303](https://github.com/ClickHouse/ClickHouse/pull/11303) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potential uninitialized memory in conversion. Example: `SELECT toIntervalSecond(now64())`. [#11311](https://github.com/ClickHouse/ClickHouse/pull/11311) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix insignificant data race in clickhouse-copier. Found by integration tests. [#11313](https://github.com/ClickHouse/ClickHouse/pull/11313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix very rare race condition in ThreadPool. [#11314](https://github.com/ClickHouse/ClickHouse/pull/11314) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix visitParamExtractRaw when extracted JSON has strings with unbalanced { or [. [#11318](https://github.com/ClickHouse/ClickHouse/pull/11318) ([Ewout](https://github.com/devwout)). +* Make writing to `MATERIALIZED VIEW` with setting `parallel_view_processing = 1` parallel again. Fixes [#10241](https://github.com/ClickHouse/ClickHouse/issues/10241). [#11330](https://github.com/ClickHouse/ClickHouse/pull/11330) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Now merges stopped before change metadata in `ALTER` queries. [#11335](https://github.com/ClickHouse/ClickHouse/pull/11335) ([alesapin](https://github.com/alesapin)). +* Fix crash in `quantilesExactWeightedArray`. [#11337](https://github.com/ClickHouse/ClickHouse/pull/11337) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in direct selects from StorageJoin (without JOIN) and wrong nullability. [#11340](https://github.com/ClickHouse/ClickHouse/pull/11340) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix potential segfault when using `Lazy` database. [#11348](https://github.com/ClickHouse/ClickHouse/pull/11348) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix support for `\G` (vertical output) in clickhouse-client in multiline mode. This closes [#9933](https://github.com/ClickHouse/ClickHouse/issues/9933). [#11350](https://github.com/ClickHouse/ClickHouse/pull/11350) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove redundant lock during parts send in ReplicatedMergeTree. [#11354](https://github.com/ClickHouse/ClickHouse/pull/11354) ([alesapin](https://github.com/alesapin)). +* Fix possible `Pipeline stuck` error for queries with external sort and limit. Fixes [#11359](https://github.com/ClickHouse/ClickHouse/issues/11359). [#11366](https://github.com/ClickHouse/ClickHouse/pull/11366) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better errors for `joinGet()` functions. [#11389](https://github.com/ClickHouse/ClickHouse/pull/11389) ([Artem Zuikov](https://github.com/4ertus2)). +* Fixed geohashesInBox with arguments outside of latitude/longitude range. [#11403](https://github.com/ClickHouse/ClickHouse/pull/11403) ([Vasily Nemkov](https://github.com/Enmk)). +* Fix error code for wrong `USING` key. [#11373](https://github.com/ClickHouse/ClickHouse/issues/11373). [#11404](https://github.com/ClickHouse/ClickHouse/pull/11404) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix crash in JOIN over LowCarinality(T) and Nullable(T). [#11380](https://github.com/ClickHouse/ClickHouse/issues/11380). [#11414](https://github.com/ClickHouse/ClickHouse/pull/11414) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix potential uninitialized memory read in MergeTree shutdown if table was not created successfully. [#11420](https://github.com/ClickHouse/ClickHouse/pull/11420) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix pointInPolygon with nan as point. Fixes [#11375](https://github.com/ClickHouse/ClickHouse/issues/11375). [#11421](https://github.com/ClickHouse/ClickHouse/pull/11421) ([Alexey Ilyukhov](https://github.com/livace)). +* Fix server crash when a column has compression codec with non-literal arguments. Fixes [#11365](https://github.com/ClickHouse/ClickHouse/issues/11365). [#11431](https://github.com/ClickHouse/ClickHouse/pull/11431) ([alesapin](https://github.com/alesapin)). +* Fix return compressed size for codecs. [#11448](https://github.com/ClickHouse/ClickHouse/pull/11448) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix wrong result in queries like `select count() from t, u`. [#11454](https://github.com/ClickHouse/ClickHouse/pull/11454) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix `Pipeline stuck` exception for `INSERT SELECT FINAL` where `SELECT` (`max_threads`>1) has multiple streams but `INSERT` has only one (`max_insert_threads`==0). [#11455](https://github.com/ClickHouse/ClickHouse/pull/11455) ([Azat Khuzhin](https://github.com/azat)). +* Fix memory leak when exception is thrown in the middle of aggregation with -State functions. This fixes [#8995](https://github.com/ClickHouse/ClickHouse/issues/8995). [#11496](https://github.com/ClickHouse/ClickHouse/pull/11496) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix race condition which may lead to an exception during table drop. It's a bit tricky and not dangerous at all. If you want an explanation, just notice me in telegram. [#11523](https://github.com/ClickHouse/ClickHouse/pull/11523) ([alesapin](https://github.com/alesapin)). +* Fix async INSERT into Distributed for prefer_localhost_replica=0 and w/o internal_replication. [#11527](https://github.com/ClickHouse/ClickHouse/pull/11527) ([Azat Khuzhin](https://github.com/azat)). +* Fix shard_num/replica_num for `` (breaks use_compact_format_in_distributed_parts_names). [#11528](https://github.com/ClickHouse/ClickHouse/pull/11528) ([Azat Khuzhin](https://github.com/azat)). +* Fix the error `Data compressed with different methods` that can happen if `min_bytes_to_use_direct_io` is enabled and PREWHERE is active and using SAMPLE or high number of threads. This fixes [#11539](https://github.com/ClickHouse/ClickHouse/issues/11539). [#11540](https://github.com/ClickHouse/ClickHouse/pull/11540) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now clickhouse-server docker container will prefer IPv6 checking server aliveness. [#11550](https://github.com/ClickHouse/ClickHouse/pull/11550) ([Ivan Starkov](https://github.com/istarkov)). +* All queries in HTTP session have had the same query_id. It is fixed. [#11578](https://github.com/ClickHouse/ClickHouse/pull/11578) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed rare segfault in `SHOW CREATE TABLE` Fixes [#11490](https://github.com/ClickHouse/ClickHouse/issues/11490). [#11579](https://github.com/ClickHouse/ClickHouse/pull/11579) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix trivial error in log message about "Mark cache size was lowered" at server startup. This closes [#11399](https://github.com/ClickHouse/ClickHouse/issues/11399). [#11589](https://github.com/ClickHouse/ClickHouse/pull/11589) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix race conditions in CREATE/DROP of different replicas of ReplicatedMergeTree. Continue to work if the table was not removed completely from ZooKeeper or not created successfully. This fixes [#11432](https://github.com/ClickHouse/ClickHouse/issues/11432). [#11592](https://github.com/ClickHouse/ClickHouse/pull/11592) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong exit code of the clickhouse-client, when exception.code() % 256 = 0. [#11601](https://github.com/ClickHouse/ClickHouse/pull/11601) ([filimonov](https://github.com/filimonov)). +* Fix error `Block structure mismatch` for queries with sampling reading from `Buffer` table. [#11602](https://github.com/ClickHouse/ClickHouse/pull/11602) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* without -q option the database does not get created at startup. [#11604](https://github.com/ClickHouse/ClickHouse/pull/11604) ([giordyb](https://github.com/giordyb)). +* Fix rare crash caused by using `Nullable` column in prewhere condition. (Probably it is connected with [#11572](https://github.com/ClickHouse/ClickHouse/issues/11572) somehow). [#11608](https://github.com/ClickHouse/ClickHouse/pull/11608) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix bloom filters for String (data skipping indices). [#11638](https://github.com/ClickHouse/ClickHouse/pull/11638) ([Azat Khuzhin](https://github.com/azat)). +* Remove trivial count query optimization if row-level security is set. In previous versions the user get total count of records in a table instead filtered. This fixes [#11352](https://github.com/ClickHouse/ClickHouse/issues/11352). [#11644](https://github.com/ClickHouse/ClickHouse/pull/11644) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add support for regular expressions with case-insensitive flags. This fixes [#11101](https://github.com/ClickHouse/ClickHouse/issues/11101) and fixes [#11506](https://github.com/ClickHouse/ClickHouse/issues/11506). [#11649](https://github.com/ClickHouse/ClickHouse/pull/11649) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix syntax hilite in CREATE USER query. [#11664](https://github.com/ClickHouse/ClickHouse/pull/11664) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error which leads to an incorrect state of `system.mutations`. It may show that whole mutation is already done but the server still has `MUTATE_PART` tasks in the replication queue and tries to execute them. This fixes [#11611](https://github.com/ClickHouse/ClickHouse/issues/11611). [#11681](https://github.com/ClickHouse/ClickHouse/pull/11681) ([alesapin](https://github.com/alesapin)). +* Fix possible `Pipeline stuck` for selects with parallel `FINAL`. Fixes [#11636](https://github.com/ClickHouse/ClickHouse/issues/11636). [#11682](https://github.com/ClickHouse/ClickHouse/pull/11682) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `LIMIT n WITH TIES` usage together with `ORDER BY` statement, which contains aliases. [#11689](https://github.com/ClickHouse/ClickHouse/pull/11689) ([Anton Popov](https://github.com/CurtizJ)). +* Pass proper timeouts when communicating with XDBC bridge. Recently timeouts were not respected when checking bridge liveness and receiving meta info. [#11690](https://github.com/ClickHouse/ClickHouse/pull/11690) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix very rare race condition in SYSTEM SYNC REPLICA. If the replicated table is created and at the same time from the separate connection another client is issuing `SYSTEM SYNC REPLICA` command on that table (this is unlikely, because another client should be aware that the table is created), it's possible to get nullptr dereference. [#11691](https://github.com/ClickHouse/ClickHouse/pull/11691) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `ORDER BY ... WITH FILL` over const columns. [#11697](https://github.com/ClickHouse/ClickHouse/pull/11697) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed LOGICAL_ERROR caused by wrong type deduction of complex literals in Values input format. [#11732](https://github.com/ClickHouse/ClickHouse/pull/11732) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Parse metadata stored in zookeeper before checking for equality. [#11739](https://github.com/ClickHouse/ClickHouse/pull/11739) ([Azat Khuzhin](https://github.com/azat)). +* Now replicated fetches will be cancelled during metadata alter. [#11744](https://github.com/ClickHouse/ClickHouse/pull/11744) ([alesapin](https://github.com/alesapin)). +* Fixes crash in special generated queries when `optimize_arithmetic_operations_in_aggregate_functions = 1`. [#11756](https://github.com/ClickHouse/ClickHouse/pull/11756) ([Ruslan](https://github.com/kamalov-ruslan)). +* Fixed `Scalar doesn't exist` exception when using `WITH ...` in `SELECT ... FROM merge_tree_table ...` [#11621](https://github.com/ClickHouse/ClickHouse/issues/11621). [#11767](https://github.com/ClickHouse/ClickHouse/pull/11767) ([Amos Bird](https://github.com/amosbird)). +* Fix using too many threads for queries. [#11788](https://github.com/ClickHouse/ClickHouse/pull/11788) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Preserve column alias with optimize_aggregators_of_group_by_keys (`optimize_aggregators_of_group_by_keys` has been introduced in [#11667](https://github.com/ClickHouse/ClickHouse/issues/11667)). [#11806](https://github.com/ClickHouse/ClickHouse/pull/11806) ([Azat Khuzhin](https://github.com/azat)). +* Fix wrong result of comparison of FixedString with constant String. This fixes [#11393](https://github.com/ClickHouse/ClickHouse/issues/11393). This bug appeared in version 20.4. [#11828](https://github.com/ClickHouse/ClickHouse/pull/11828) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't allow arrayJoin inside higher order functions. It was leading to broken protocol synchronization. This closes [#3933](https://github.com/ClickHouse/ClickHouse/issues/3933). [#11846](https://github.com/ClickHouse/ClickHouse/pull/11846) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Build/Testing/Packaging Improvement +* Add new build for query tests using pytest framework. [#10039](https://github.com/ClickHouse/ClickHouse/pull/10039) ([Ivan](https://github.com/abyss7)). +* Fix FreeBSD build. [#10150](https://github.com/ClickHouse/ClickHouse/pull/10150) ([Ivan](https://github.com/abyss7)). +* Fix UBSan report in Decimal parse. This fixes [#7540](https://github.com/ClickHouse/ClickHouse/issues/7540). [#10512](https://github.com/ClickHouse/ClickHouse/pull/10512) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adding fuzzers and preparing for oss-fuzz integration. [#10546](https://github.com/ClickHouse/ClickHouse/pull/10546) ([kyprizel](https://github.com/kyprizel)). +* Enable ThinLTO for clang builds, continuation of https://github.com/ClickHouse/ClickHouse/pull/10435. [#10585](https://github.com/ClickHouse/ClickHouse/pull/10585) ([Amos Bird](https://github.com/amosbird)). +* Increasing timeout when opening a client in tests/queries/0_stateless/helpers/client.py. [#10599](https://github.com/ClickHouse/ClickHouse/pull/10599) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fixing hard coded timeouts in new live view tests. [#10604](https://github.com/ClickHouse/ClickHouse/pull/10604) ([vzakaznikov](https://github.com/vzakaznikov)). +* Lower memory usage in tests. It may fix the issue that "address sanitizer is out of memory" in stress test. [#10617](https://github.com/ClickHouse/ClickHouse/pull/10617) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `capnproto` version check for `capnp::UnalignedFlatArrayMessageReader`. [#10618](https://github.com/ClickHouse/ClickHouse/pull/10618) ([Matwey V. Kornilov](https://github.com/matwey)). +* Added auto-generated machine-readable file with list of stable versions. [#10628](https://github.com/ClickHouse/ClickHouse/pull/10628) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update LZ4 to the latest dev branch. It may fix the error under UBSan. [#10630](https://github.com/ClickHouse/ClickHouse/pull/10630) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in LZ4 library. [#10631](https://github.com/ClickHouse/ClickHouse/pull/10631) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to use lld to link blobs (resources). [#10632](https://github.com/ClickHouse/ClickHouse/pull/10632) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove external call to `ld` (bfd) linker during tzdata processing in compile time. [#10634](https://github.com/ClickHouse/ClickHouse/pull/10634) ([alesapin](https://github.com/alesapin)). +* Fix UBSan report (adding zero to nullptr) in HashTable that appeared after migration to clang-10. [#10638](https://github.com/ClickHouse/ClickHouse/pull/10638) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix OOM in ASan stress test. [#10646](https://github.com/ClickHouse/ClickHouse/pull/10646) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixing and re-enabling 00979_live_view_watch_continuous_aggregates.py test. [#10658](https://github.com/ClickHouse/ClickHouse/pull/10658) ([vzakaznikov](https://github.com/vzakaznikov)). +* Update zstd to 1.4.4. It has some minor improvements in performance and compression ratio. If you run replicas with different versions of ClickHouse you may see reasonable error messages `Data after merge is not byte-identical to data on another replicas.` with explanation. These messages are Ok and you should not worry. [#10663](https://github.com/ClickHouse/ClickHouse/pull/10663) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Volumes and storages refactoring. [#10666](https://github.com/ClickHouse/ClickHouse/pull/10666) ([Gleb Novikov](https://github.com/NanoBjorn)). +* Trying to fix tests/queries/0_stateless/01246_insert_into_watch_live_view.py test. [#10670](https://github.com/ClickHouse/ClickHouse/pull/10670) ([vzakaznikov](https://github.com/vzakaznikov)). +* Update instruction to install RPM packages. This was suggested by Denis (TG login @ldviolet) and implemented by Arkady Shejn. [#10707](https://github.com/ClickHouse/ClickHouse/pull/10707) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update cross-builds to use clang-10 compiler. [#10724](https://github.com/ClickHouse/ClickHouse/pull/10724) ([Ivan](https://github.com/abyss7)). +* Fix performance test errors. [#10766](https://github.com/ClickHouse/ClickHouse/pull/10766) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix performance tests errors, part 2. [#10773](https://github.com/ClickHouse/ClickHouse/pull/10773) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Restore a patch that was accidentially deleted in [#10396](https://github.com/ClickHouse/ClickHouse/issues/10396). [#10774](https://github.com/ClickHouse/ClickHouse/pull/10774) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Suppressions of warnings from libraries was mistakenly declared as public in [#10396](https://github.com/ClickHouse/ClickHouse/issues/10396). [#10776](https://github.com/ClickHouse/ClickHouse/pull/10776) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable extra warnings for base, utils, programs. [#10779](https://github.com/ClickHouse/ClickHouse/pull/10779) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* GRPC make couldn't find protobuf files, changed make file by adding the right link. [#10794](https://github.com/ClickHouse/ClickHouse/pull/10794) ([mnkonkova](https://github.com/mnkonkova)). +* Add MSan suppression for MariaDB Client library. [#10800](https://github.com/ClickHouse/ClickHouse/pull/10800) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix (false) MSan report in MergeTreeIndexFullText. The issue first appeared in [#9968](https://github.com/ClickHouse/ClickHouse/issues/9968). [#10801](https://github.com/ClickHouse/ClickHouse/pull/10801) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix obvious race condition in "Split build smoke test" check. [#10820](https://github.com/ClickHouse/ClickHouse/pull/10820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better cooperation with sanitizers. Print information about query_id in the message of sanitizer failure. [#10832](https://github.com/ClickHouse/ClickHouse/pull/10832) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added more asserts in columns code. [#10833](https://github.com/ClickHouse/ClickHouse/pull/10833) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Prepare to add MSan and UBSan stress tests. [#10871](https://github.com/ClickHouse/ClickHouse/pull/10871) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Database is recreated for every test. This improves separation of tests. [#10902](https://github.com/ClickHouse/ClickHouse/pull/10902) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added a test for empty external data. [#10926](https://github.com/ClickHouse/ClickHouse/pull/10926) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Return tzdata to build images and as dependency to .deb package. [#10929](https://github.com/ClickHouse/ClickHouse/pull/10929) ([alesapin](https://github.com/alesapin)). +* Fix non-deterministic test. [#10989](https://github.com/ClickHouse/ClickHouse/pull/10989) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Wait for odbc-bridge with exponential backoff. Previous wait time of 200 ms was not enough in our CI environment. [#10990](https://github.com/ClickHouse/ClickHouse/pull/10990) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable clang-tidy for programs and utils. [#10991](https://github.com/ClickHouse/ClickHouse/pull/10991) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add ability to run zookeeper in integration tests over tmpfs. [#11002](https://github.com/ClickHouse/ClickHouse/pull/11002) ([alesapin](https://github.com/alesapin)). +* Fixing 00979_live_view_watch_continuous_aggregates test. [#11024](https://github.com/ClickHouse/ClickHouse/pull/11024) ([vzakaznikov](https://github.com/vzakaznikov)). +* Make `system_tables_lazy_load` false by default. [#11029](https://github.com/ClickHouse/ClickHouse/pull/11029) ([Azat Khuzhin](https://github.com/azat)). +* Add performance test for non-constant polygons. [#11141](https://github.com/ClickHouse/ClickHouse/pull/11141) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Create root S3 bucket for tests before any CH instance is started. [#11142](https://github.com/ClickHouse/ClickHouse/pull/11142) ([Pavel Kovalenko](https://github.com/Jokser)). +* Enable performance test that was not working. [#11158](https://github.com/ClickHouse/ClickHouse/pull/11158) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve build scripts for protobuf & gRPC. [#11172](https://github.com/ClickHouse/ClickHouse/pull/11172) ([Vitaly Baranov](https://github.com/vitlibar)). +* Split /programs/server into actual program and library. [#11186](https://github.com/ClickHouse/ClickHouse/pull/11186) ([Ivan](https://github.com/abyss7)). +* Now parts of linker command for `cctz` library will not be shuffled with other libraries. [#11213](https://github.com/ClickHouse/ClickHouse/pull/11213) ([alesapin](https://github.com/alesapin)). +* Fix several non significant errors in unit tests. [#11262](https://github.com/ClickHouse/ClickHouse/pull/11262) ([alesapin](https://github.com/alesapin)). +* Add a test for Join table engine from @donmikel. This closes [#9158](https://github.com/ClickHouse/ClickHouse/issues/9158). [#11265](https://github.com/ClickHouse/ClickHouse/pull/11265) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Repeat test in CI if `curl` invocation was timed out. It is possible due to system hangups for 10+ seconds that are typical in our CI infrastructure. This fixes [#11267](https://github.com/ClickHouse/ClickHouse/issues/11267). [#11268](https://github.com/ClickHouse/ClickHouse/pull/11268) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potentially flacky test `00731_long_merge_tree_select_opened_files.sh`. It does not fail frequently but we have discovered potential race condition in this test while experimenting with ThreadFuzzer: [#9814](https://github.com/ClickHouse/ClickHouse/issues/9814) See [link](https://clickhouse-test-reports.s3.yandex.net/9814/40e3023e215df22985d275bf85f4d2290897b76b/functional_stateless_tests_(unbundled).html#fail1) for the example. [#11270](https://github.com/ClickHouse/ClickHouse/pull/11270) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now clickhouse-test check the server aliveness before tests run. [#11285](https://github.com/ClickHouse/ClickHouse/pull/11285) ([alesapin](https://github.com/alesapin)). +* Emit a warning if server was build in debug or with sanitizers. [#11304](https://github.com/ClickHouse/ClickHouse/pull/11304) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better check for hung queries in clickhouse-test. [#11321](https://github.com/ClickHouse/ClickHouse/pull/11321) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove redundant timeout from integration test `test_insertion_sync_fails_with_timeout`. [#11343](https://github.com/ClickHouse/ClickHouse/pull/11343) ([alesapin](https://github.com/alesapin)). +* Add support for unit tests run with UBSan. [#11345](https://github.com/ClickHouse/ClickHouse/pull/11345) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix several flaky integration tests. [#11355](https://github.com/ClickHouse/ClickHouse/pull/11355) ([alesapin](https://github.com/alesapin)). +* Refactor CMake build files. [#11390](https://github.com/ClickHouse/ClickHouse/pull/11390) ([Ivan](https://github.com/abyss7)). +* Leave only unit_tests_dbms in deb build. [#11429](https://github.com/ClickHouse/ClickHouse/pull/11429) ([Ilya Yatsishin](https://github.com/qoega)). +* Increase ccache size for builds in CI. [#11450](https://github.com/ClickHouse/ClickHouse/pull/11450) ([alesapin](https://github.com/alesapin)). +* Speed up build by removing old example programs. Also found some orphan functional test. [#11486](https://github.com/ClickHouse/ClickHouse/pull/11486) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix missed include for `std::move` used at line 17. [#11525](https://github.com/ClickHouse/ClickHouse/pull/11525) ([Matwey V. Kornilov](https://github.com/matwey)). +* Added a random sampling of instances where copier is executed. It is needed to avoid `Too many simultaneous queries` error. Also increased timeout and decreased fault probability. [#11573](https://github.com/ClickHouse/ClickHouse/pull/11573) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Removes unused imports from HTTPHandlerFactory. [#11660](https://github.com/ClickHouse/ClickHouse/pull/11660) ([Bharat Nallan](https://github.com/bharatnc)). +* Don't allow tests with "fail" substring in their names because it makes looking at the tests results in browser less convenient when you type Ctrl+F and search for "fail". [#11817](https://github.com/ClickHouse/ClickHouse/pull/11817) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added a test to ensure that mutations continue to work after FREEZE query. [#11820](https://github.com/ClickHouse/ClickHouse/pull/11820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Print compiler build id in crash messages. It will make us slightly more certain about what binary has crashed. Added new function `buildId`. [#11824](https://github.com/ClickHouse/ClickHouse/pull/11824) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove dependency on `tzdata`: do not fail if `/usr/share/zoneinfo` directory does not exist. Note that all timezones work in ClickHouse even without tzdata installed in system. [#11827](https://github.com/ClickHouse/ClickHouse/pull/11827) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NO CL CATEGORY + +* * Not for changelog. [#10985](https://github.com/ClickHouse/ClickHouse/pull/10985) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Bump numpy from 1.18.3 to 1.18.4 in /docs/tools'. [#10648](https://github.com/ClickHouse/ClickHouse/pull/10648) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: '[ImgBot] Optimize images'. [#10796](https://github.com/ClickHouse/ClickHouse/pull/10796) ([imgbot[bot]](https://github.com/apps/imgbot)). +* NO CL ENTRY: 'Bump mkdocs from 1.1 to 1.1.1 in /docs/tools'. [#10877](https://github.com/ClickHouse/ClickHouse/pull/10877) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump mkdocs-macros-plugin from 0.4.6 to 0.4.7 in /docs/tools'. [#10878](https://github.com/ClickHouse/ClickHouse/pull/10878) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump mkdocs from 1.1.1 to 1.1.2 in /docs/tools'. [#10938](https://github.com/ClickHouse/ClickHouse/pull/10938) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump protobuf from 3.11.3 to 3.12.0 in /docs/tools'. [#10995](https://github.com/ClickHouse/ClickHouse/pull/10995) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump beautifulsoup4 from 4.9.0 to 4.9.1 in /docs/tools'. [#10996](https://github.com/ClickHouse/ClickHouse/pull/10996) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump soupsieve from 2.0 to 2.0.1 in /docs/tools'. [#10997](https://github.com/ClickHouse/ClickHouse/pull/10997) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump mkdocs-macros-plugin from 0.4.7 to 0.4.9 in /docs/tools'. [#11064](https://github.com/ClickHouse/ClickHouse/pull/11064) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump protobuf from 3.12.0 to 3.12.1 in /docs/tools'. [#11093](https://github.com/ClickHouse/ClickHouse/pull/11093) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump six from 1.14.0 to 1.15.0 in /docs/tools'. [#11129](https://github.com/ClickHouse/ClickHouse/pull/11129) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump protobuf from 3.12.1 to 3.12.2 in /docs/tools'. [#11241](https://github.com/ClickHouse/ClickHouse/pull/11241) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump numpy from 1.18.4 to 1.18.5 in /docs/tools'. [#11427](https://github.com/ClickHouse/ClickHouse/pull/11427) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump livereload from 2.6.1 to 2.6.2 in /docs/tools'. [#11502](https://github.com/ClickHouse/ClickHouse/pull/11502) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump certifi from 2020.4.5.1 to 2020.4.5.2 in /docs/tools'. [#11503](https://github.com/ClickHouse/ClickHouse/pull/11503) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump googletrans from 2.4.0 to 3.0.0 in /docs/tools'. [#11675](https://github.com/ClickHouse/ClickHouse/pull/11675) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump requests from 2.23.0 to 2.24.0 in /docs/tools'. [#11750](https://github.com/ClickHouse/ClickHouse/pull/11750) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). + +#### New Feature: function randomStringUTF8 + +* Added function randomStringUTF8. [#10972](https://github.com/ClickHouse/ClickHouse/pull/10972) ([Andrei Nekrashevich](https://github.com/axolm)). + diff --git a/docs/changelogs/v20.5.2.7-stable.md b/docs/changelogs/v20.5.2.7-stable.md new file mode 100644 index 00000000000..10b598cb9ba --- /dev/null +++ b/docs/changelogs/v20.5.2.7-stable.md @@ -0,0 +1,33 @@ +### ClickHouse release v20.5.2.7-stable FIXME as compared to v20.5.1.3833-prestable + +#### New Feature +* Add `Alter table drop replica replica_name` support. This fixes [#7080](https://github.com/ClickHouse/ClickHouse/issues/7080). [#10679](https://github.com/ClickHouse/ClickHouse/pull/10679) ([sundyli](https://github.com/sundy-li)). + +#### Improvement +* Add number of errors to ignore while choosing replicas (`distributed_replica_error_ignore`). [#11669](https://github.com/ClickHouse/ClickHouse/pull/11669) ([Azat Khuzhin](https://github.com/azat)). +* Multiversion metadata for storages without structure locks. [#11745](https://github.com/ClickHouse/ClickHouse/pull/11745) ([alesapin](https://github.com/alesapin)). +* Slightly relax the validation of ODBC connection string. If the hostname or username contains only word characters along with `.` and `-`, don't put it into curly braces. It is needed, because some ODBC drivers (e.g. PostgreSQL) don't understand when hostname is enclosed in curly braces. [#11845](https://github.com/ClickHouse/ClickHouse/pull/11845) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support `SIGNED` and `UNSIGNED` modifiers of standard integer types (`BIGINT`, `INT`, ...) for compatibility with MySQL. [#11858](https://github.com/ClickHouse/ClickHouse/pull/11858) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Allow to use `sumWithOverflow` as `SimpleAggregateFunction`. Closes [#8053](https://github.com/ClickHouse/ClickHouse/issues/8053). [#11865](https://github.com/ClickHouse/ClickHouse/pull/11865) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add FixedString support in Hashing functions. [#11878](https://github.com/ClickHouse/ClickHouse/pull/11878) ([flynn](https://github.com/ucasfl)). + +#### Bug Fix +* Fix unexpected behaviour of queries like `SELECT *, xyz.*` which were success while an error expected. [#11753](https://github.com/ClickHouse/ClickHouse/pull/11753) ([hexiaoting](https://github.com/hexiaoting)). +* Fix wrong result for `if()` with NULLs in condition. [#11807](https://github.com/ClickHouse/ClickHouse/pull/11807) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix memory accounting via HTTP interface (can be significant with `wait_end_of_query=1`). [#11840](https://github.com/ClickHouse/ClickHouse/pull/11840) ([Azat Khuzhin](https://github.com/azat)). +* Fix rare crash caused by using `Nullable` column in prewhere condition. Continuation of [#11608](https://github.com/ClickHouse/ClickHouse/issues/11608). [#11869](https://github.com/ClickHouse/ClickHouse/pull/11869) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#11965](https://github.com/ClickHouse/ClickHouse/issues/11965): Fix potential floating point exception when parsing DateTime64. This fixes [#11374](https://github.com/ClickHouse/ClickHouse/issues/11374). [#11875](https://github.com/ClickHouse/ClickHouse/pull/11875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potential floating point exception when parsing DateTime64. This fixes [#11374](https://github.com/ClickHouse/ClickHouse/issues/11374). [#11875](https://github.com/ClickHouse/ClickHouse/pull/11875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#11963](https://github.com/ClickHouse/ClickHouse/issues/11963): Use the correct current database for checking access rights after statement `USE database`. [#11920](https://github.com/ClickHouse/ClickHouse/pull/11920) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#12059](https://github.com/ClickHouse/ClickHouse/issues/12059): Fix incorrect comparison of tuples with `Nullable` columns. Fixes [#11985](https://github.com/ClickHouse/ClickHouse/issues/11985). [#12039](https://github.com/ClickHouse/ClickHouse/pull/12039) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Build/Testing/Packaging Improvement +* Add simple GitHub hook script for the serverless environment. [#11605](https://github.com/ClickHouse/ClickHouse/pull/11605) ([alesapin](https://github.com/alesapin)). +* Send logs to client on fatal errors if possible. This will make test results more readable. [#11826](https://github.com/ClickHouse/ClickHouse/pull/11826) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow ClickHouse to run on Android. [#11894](https://github.com/ClickHouse/ClickHouse/pull/11894) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Merging [#10679](https://github.com/ClickHouse/ClickHouse/issues/10679)'. [#11896](https://github.com/ClickHouse/ClickHouse/pull/11896) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Cherry pick [#11875](https://github.com/ClickHouse/ClickHouse/issues/11875) to 20.5: Fix strange and wrong code around DateTime64'. [#11958](https://github.com/ClickHouse/ClickHouse/pull/11958) ([Ivan](https://github.com/abyss7)). + diff --git a/docs/changelogs/v20.5.3.27-stable.md b/docs/changelogs/v20.5.3.27-stable.md new file mode 100644 index 00000000000..58b265243ef --- /dev/null +++ b/docs/changelogs/v20.5.3.27-stable.md @@ -0,0 +1,60 @@ +### ClickHouse release v20.5.3.27-stable FIXME as compared to v20.5.2.7-stable + +#### Improvement +* Moved useless S3 logging to TRACE level. [#12067](https://github.com/ClickHouse/ClickHouse/pull/12067) ([Vladimir Chebotarev](https://github.com/excitoon)). + +#### Bug Fix +* Backported in [#11967](https://github.com/ClickHouse/ClickHouse/issues/11967): Fix unexpected behaviour of queries like `SELECT *, xyz.*` which were success while an error expected. [#11753](https://github.com/ClickHouse/ClickHouse/pull/11753) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#11968](https://github.com/ClickHouse/ClickHouse/issues/11968): Fix memory accounting via HTTP interface (can be significant with `wait_end_of_query=1`). [#11840](https://github.com/ClickHouse/ClickHouse/pull/11840) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#12354](https://github.com/ClickHouse/ClickHouse/issues/12354): Fixed bug with no moves when changing storage policy from default one. [#11893](https://github.com/ClickHouse/ClickHouse/pull/11893) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fixed bug with no moves when changing storage policy from default one. [#11893](https://github.com/ClickHouse/ClickHouse/pull/11893) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#12244](https://github.com/ClickHouse/ClickHouse/issues/12244): Fix wrong setting name in log message at server startup. [#11997](https://github.com/ClickHouse/ClickHouse/pull/11997) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Partial revokes work correctly in complex cases as well, for example. [#12002](https://github.com/ClickHouse/ClickHouse/pull/12002) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#12245](https://github.com/ClickHouse/ClickHouse/issues/12245): Fix potential floating point exception. This closes [#11378](https://github.com/ClickHouse/ClickHouse/issues/11378). [#12005](https://github.com/ClickHouse/ClickHouse/pull/12005) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12249](https://github.com/ClickHouse/ClickHouse/issues/12249): Fix potential array size overflow in generateRandom that may lead to crash. This fixes [#11371](https://github.com/ClickHouse/ClickHouse/issues/11371). [#12013](https://github.com/ClickHouse/ClickHouse/pull/12013) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12251](https://github.com/ClickHouse/ClickHouse/issues/12251): A query with function `neighbor` as the only returned expression may return empty result if the function is called with offset `-9223372036854775808`. This fixes [#11367](https://github.com/ClickHouse/ClickHouse/issues/11367). [#12019](https://github.com/ClickHouse/ClickHouse/pull/12019) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12272](https://github.com/ClickHouse/ClickHouse/issues/12272): Do not mention in changelog, because the bug did not come to release. Fix potential crash when doing ORDER BY multiple columns with specified COLLATE on one of the column when this column is constant. This fixes [#11379](https://github.com/ClickHouse/ClickHouse/issues/11379). The bug was introduced in [#11006](https://github.com/ClickHouse/ClickHouse/issues/11006) in version 20.5. [#12020](https://github.com/ClickHouse/ClickHouse/pull/12020) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12241](https://github.com/ClickHouse/ClickHouse/issues/12241): Fix wrong result and potential crash when invoking function `if` with arguments of type `FixedString` with different sizes. This fixes [#11362](https://github.com/ClickHouse/ClickHouse/issues/11362). [#12021](https://github.com/ClickHouse/ClickHouse/pull/12021) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12230](https://github.com/ClickHouse/ClickHouse/issues/12230): Fix crash in JOIN with LowCardinality type with `join_algorithm=partial_merge`. [#12035](https://github.com/ClickHouse/ClickHouse/pull/12035) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#12231](https://github.com/ClickHouse/ClickHouse/issues/12231): Fix constraints check if constraint is a constant expression. This fixes [#11360](https://github.com/ClickHouse/ClickHouse/issues/11360). [#12042](https://github.com/ClickHouse/ClickHouse/pull/12042) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12232](https://github.com/ClickHouse/ClickHouse/issues/12232): Make `topK` aggregate function return Enum for Enum types. This fixes [#3740](https://github.com/ClickHouse/ClickHouse/issues/3740). [#12043](https://github.com/ClickHouse/ClickHouse/pull/12043) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12233](https://github.com/ClickHouse/ClickHouse/issues/12233): Parse tables metadata in parallel when loading database. This fixes slow server startup when there are large number of tables. [#12045](https://github.com/ClickHouse/ClickHouse/pull/12045) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#12234](https://github.com/ClickHouse/ClickHouse/issues/12234): Fix error `Cannot capture column` for higher-order functions with `Tuple(LowCardinality)` argument. Fixes [#9766](https://github.com/ClickHouse/ClickHouse/issues/9766). [#12055](https://github.com/ClickHouse/ClickHouse/pull/12055) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix error `Expected single dictionary argument for function` for function `defaultValueOfArgumentType` with `LowCardinality` type. Fixes [#11808](https://github.com/ClickHouse/ClickHouse/issues/11808). [#12056](https://github.com/ClickHouse/ClickHouse/pull/12056) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#12235](https://github.com/ClickHouse/ClickHouse/issues/12235): Fix error `Expected single dictionary argument for function` for function `defaultValueOfArgumentType` with `LowCardinality` type. Fixes [#11808](https://github.com/ClickHouse/ClickHouse/issues/11808). [#12056](https://github.com/ClickHouse/ClickHouse/pull/12056) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#12236](https://github.com/ClickHouse/ClickHouse/issues/12236): Fix possible crash while using wrong type for `PREWHERE`. Fixes [#12053](https://github.com/ClickHouse/ClickHouse/issues/12053), [#12060](https://github.com/ClickHouse/ClickHouse/issues/12060). [#12060](https://github.com/ClickHouse/ClickHouse/pull/12060) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#12237](https://github.com/ClickHouse/ClickHouse/issues/12237): Fix SIGSEGV in StorageKafka on DROP TABLE. [#12075](https://github.com/ClickHouse/ClickHouse/pull/12075) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#12239](https://github.com/ClickHouse/ClickHouse/issues/12239): Fix empty `result_rows` and `result_bytes` metrics in `system.quey_log` for selects. Fixes [#11595](https://github.com/ClickHouse/ClickHouse/issues/11595). [#12089](https://github.com/ClickHouse/ClickHouse/pull/12089) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#12240](https://github.com/ClickHouse/ClickHouse/issues/12240): Fix segfault with `-StateResample` combinators. [#12092](https://github.com/ClickHouse/ClickHouse/pull/12092) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#12242](https://github.com/ClickHouse/ClickHouse/issues/12242): Format `Parquet` now properly works with `LowCardinality` and `LowCardinality(Nullable)` types. Fixes [#12086](https://github.com/ClickHouse/ClickHouse/issues/12086), [#8406](https://github.com/ClickHouse/ClickHouse/issues/8406). [#12108](https://github.com/ClickHouse/ClickHouse/pull/12108) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#12260](https://github.com/ClickHouse/ClickHouse/issues/12260): Fix handling dependency of table with ENGINE=Dictionary on dictionary. This fixes [#10994](https://github.com/ClickHouse/ClickHouse/issues/10994). This fixes [#10397](https://github.com/ClickHouse/ClickHouse/issues/10397). [#12116](https://github.com/ClickHouse/ClickHouse/pull/12116) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#12243](https://github.com/ClickHouse/ClickHouse/issues/12243): Avoid "There is no query" exception for materialized views with joins or with subqueries attached to system logs (system.query_log, metric_log, etc) or to engine=Buffer underlying table. [#12120](https://github.com/ClickHouse/ClickHouse/pull/12120) ([filimonov](https://github.com/filimonov)). +* Backported in [#12392](https://github.com/ClickHouse/ClickHouse/issues/12392): Fix bug which leads to incorrect table metadata in ZooKeepeer for ReplicatedVersionedCollapsingMergeTree tables. Fixes [#12093](https://github.com/ClickHouse/ClickHouse/issues/12093). [#12121](https://github.com/ClickHouse/ClickHouse/pull/12121) ([alesapin](https://github.com/alesapin)). +* Backported in [#12246](https://github.com/ClickHouse/ClickHouse/issues/12246): Normalize "pid" file handling. In previous versions the server may refuse to start if it was killed without proper shutdown and if there is another process that has the same pid as previously runned server. Also pid file may be removed in unsuccessful server startup even if there is another server running. This fixes [#3501](https://github.com/ClickHouse/ClickHouse/issues/3501). [#12133](https://github.com/ClickHouse/ClickHouse/pull/12133) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12247](https://github.com/ClickHouse/ClickHouse/issues/12247): Fix potential infinite loop in `greatCircleDistance`, `geoDistance`. This fixes [#12117](https://github.com/ClickHouse/ClickHouse/issues/12117). [#12137](https://github.com/ClickHouse/ClickHouse/pull/12137) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12248](https://github.com/ClickHouse/ClickHouse/issues/12248): Fix potential overflow in integer division. This fixes [#12119](https://github.com/ClickHouse/ClickHouse/issues/12119). [#12140](https://github.com/ClickHouse/ClickHouse/pull/12140) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12252](https://github.com/ClickHouse/ClickHouse/issues/12252): Fix bad code in redundant ORDER BY optimization. The bug was introduced in [#10067](https://github.com/ClickHouse/ClickHouse/issues/10067). [#12148](https://github.com/ClickHouse/ClickHouse/pull/12148) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12360](https://github.com/ClickHouse/ClickHouse/issues/12360): Fix transform of query to send to external DBMS (e.g. MySQL, ODBC) in presense of aliases. This fixes [#12032](https://github.com/ClickHouse/ClickHouse/issues/12032). [#12151](https://github.com/ClickHouse/ClickHouse/pull/12151) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12253](https://github.com/ClickHouse/ClickHouse/issues/12253): Fix wrong logic in ALTER DELETE that leads to deleting of records when condition evaluates to NULL. This fixes [#9088](https://github.com/ClickHouse/ClickHouse/issues/9088). This closes [#12106](https://github.com/ClickHouse/ClickHouse/issues/12106). [#12153](https://github.com/ClickHouse/ClickHouse/pull/12153) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12259](https://github.com/ClickHouse/ClickHouse/issues/12259): Don't split the dictionary source's table name into schema and table name itself if ODBC connection doesn't support schema. [#12165](https://github.com/ClickHouse/ClickHouse/pull/12165) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#12366](https://github.com/ClickHouse/ClickHouse/issues/12366): Fix dictGet arguments check during GROUP BY injective functions elimination. [#12179](https://github.com/ClickHouse/ClickHouse/pull/12179) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#12367](https://github.com/ClickHouse/ClickHouse/issues/12367): Cap max_memory_usage* limits to the process resident memory. [#12182](https://github.com/ClickHouse/ClickHouse/pull/12182) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#12352](https://github.com/ClickHouse/ClickHouse/issues/12352): Fixed logical functions for UInt8 values when they are not equal to 0 or 1. [#12196](https://github.com/ClickHouse/ClickHouse/pull/12196) ([Alexander Kazakov](https://github.com/Akazz)). +* Backported in [#12369](https://github.com/ClickHouse/ClickHouse/issues/12369): Fixed behaviour on reaching redirect limit in request to S3 storage. [#12256](https://github.com/ClickHouse/ClickHouse/pull/12256) ([ianton-ru](https://github.com/ianton-ru)). +* Backported in [#12381](https://github.com/ClickHouse/ClickHouse/issues/12381): Not for changelog. Cherry-pick after [#12196](https://github.com/ClickHouse/ClickHouse/issues/12196). [#12271](https://github.com/ClickHouse/ClickHouse/pull/12271) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12370](https://github.com/ClickHouse/ClickHouse/issues/12370): Implement conversions to the common type for LowCardinality types. This allows to execute UNION ALL of tables with columns of LowCardinality and other columns. This fixes [#8212](https://github.com/ClickHouse/ClickHouse/issues/8212). This fixes [#4342](https://github.com/ClickHouse/ClickHouse/issues/4342). [#12275](https://github.com/ClickHouse/ClickHouse/pull/12275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12368](https://github.com/ClickHouse/ClickHouse/issues/12368): The function `arrayFill` worked incorrectly for empty arrays that may lead to crash. This fixes [#12263](https://github.com/ClickHouse/ClickHouse/issues/12263). [#12279](https://github.com/ClickHouse/ClickHouse/pull/12279) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12362](https://github.com/ClickHouse/ClickHouse/issues/12362): Fix typo in setting name. [#12292](https://github.com/ClickHouse/ClickHouse/pull/12292) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12363](https://github.com/ClickHouse/ClickHouse/issues/12363): Some threads might randomly hang for a few seconds during DNS cache updating. It's fixed. [#12296](https://github.com/ClickHouse/ClickHouse/pull/12296) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#12364](https://github.com/ClickHouse/ClickHouse/issues/12364): Fix TTL after renaming column, on which depends TTL expression. [#12304](https://github.com/ClickHouse/ClickHouse/pull/12304) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#12365](https://github.com/ClickHouse/ClickHouse/issues/12365): Avoid "bad cast" exception when there is an expression that filters data by virtual columns (like `_table` in `Merge` tables) or by "index" columns in system tables such as filtering by database name when querying from `system.tables`, and this expression returns `Nullable` type. This fixes [#12166](https://github.com/ClickHouse/ClickHouse/issues/12166). [#12305](https://github.com/ClickHouse/ClickHouse/pull/12305) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12375](https://github.com/ClickHouse/ClickHouse/issues/12375): Fix order of columns in `WITH FILL` modifier. Previously order of columns of `ORDER BY` statement wasn't respected. [#12306](https://github.com/ClickHouse/ClickHouse/pull/12306) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#12385](https://github.com/ClickHouse/ClickHouse/issues/12385): Fix very rare race condition in ReplicatedMergeTreeQueue. [#12315](https://github.com/ClickHouse/ClickHouse/pull/12315) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Build/Testing/Packaging Improvement +* Backported in [#12254](https://github.com/ClickHouse/ClickHouse/issues/12254): Install `ca-certificates` before the first `apt-get update` in Dockerfile. [#12095](https://github.com/ClickHouse/ClickHouse/pull/12095) ([Ivan Blinkov](https://github.com/blinkov)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Cherry pick [#12056](https://github.com/ClickHouse/ClickHouse/issues/12056) to 20.5: Fix defaultValueOfArgumentType'. [#12205](https://github.com/ClickHouse/ClickHouse/pull/12205) ([robot-clickhouse](https://github.com/robot-clickhouse)). + diff --git a/docs/changelogs/v20.5.4.40-stable.md b/docs/changelogs/v20.5.4.40-stable.md new file mode 100644 index 00000000000..7dbf555ad28 --- /dev/null +++ b/docs/changelogs/v20.5.4.40-stable.md @@ -0,0 +1,32 @@ +### ClickHouse release v20.5.4.40-stable FIXME as compared to v20.5.3.27-stable + +#### Performance Improvement +* Backported in [#12929](https://github.com/ClickHouse/ClickHouse/issues/12929): Fix "[#10574](https://github.com/ClickHouse/ClickHouse/issues/10574) Index not used for IN operator with literals", performance regression introduced around v19.3. [#12062](https://github.com/ClickHouse/ClickHouse/pull/12062) ([nvartolomei](https://github.com/nvartolomei)). + +#### Bug Fix +* Backported in [#12830](https://github.com/ClickHouse/ClickHouse/issues/12830): Fix performance for selects with `UNION` caused by wrong limit for the total number of threads. Fixes [#12030](https://github.com/ClickHouse/ClickHouse/issues/12030). [#12103](https://github.com/ClickHouse/ClickHouse/pull/12103) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#12836](https://github.com/ClickHouse/ClickHouse/issues/12836): Fixed the behaviour when `SummingMergeTree` engine sums up columns from partition key. Added an exception in case of explicit definition of columns to sum which intersects with partition key columns. This fixes [#7867](https://github.com/ClickHouse/ClickHouse/issues/7867). [#12173](https://github.com/ClickHouse/ClickHouse/pull/12173) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#12833](https://github.com/ClickHouse/ClickHouse/issues/12833): Fixed the behaviour when during multiple sequential inserts in `StorageFile` header for some special types was written more than once. This fixed [#6155](https://github.com/ClickHouse/ClickHouse/issues/6155). [#12197](https://github.com/ClickHouse/ClickHouse/pull/12197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#12895](https://github.com/ClickHouse/ClickHouse/issues/12895): kafka: fix SIGSEGV if there is an message with error in the middle of the batch. [#12302](https://github.com/ClickHouse/ClickHouse/pull/12302) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#12905](https://github.com/ClickHouse/ClickHouse/issues/12905): Fix TOTALS/ROLLUP/CUBE for aggregate functions with `-State` and `Nullable` arguments. This fixes [#12163](https://github.com/ClickHouse/ClickHouse/issues/12163). [#12376](https://github.com/ClickHouse/ClickHouse/pull/12376) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12907](https://github.com/ClickHouse/ClickHouse/issues/12907): Allow to CLEAR column even if there are depending DEFAULT expressions. This fixes [#12333](https://github.com/ClickHouse/ClickHouse/issues/12333). [#12378](https://github.com/ClickHouse/ClickHouse/pull/12378) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12909](https://github.com/ClickHouse/ClickHouse/issues/12909): Avoid exception when negative or floating point constant is used in WHERE condition for indexed tables. This fixes [#11905](https://github.com/ClickHouse/ClickHouse/issues/11905). [#12384](https://github.com/ClickHouse/ClickHouse/pull/12384) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12897](https://github.com/ClickHouse/ClickHouse/issues/12897): Fix crash in JOIN with dictionary when we are joining over expression of dictionary key: `t JOIN dict ON expr(dict.id) = t.id`. Disable dictionary join optimisation for this case. [#12458](https://github.com/ClickHouse/ClickHouse/pull/12458) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#13004](https://github.com/ClickHouse/ClickHouse/issues/13004): Fixed performance issue, while reading from compact parts. [#12492](https://github.com/ClickHouse/ClickHouse/pull/12492) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#12892](https://github.com/ClickHouse/ClickHouse/issues/12892): Fixing race condition in live view tables which could cause data duplication. [#12519](https://github.com/ClickHouse/ClickHouse/pull/12519) ([vzakaznikov](https://github.com/vzakaznikov)). +* Backported in [#12888](https://github.com/ClickHouse/ClickHouse/issues/12888): Now ClickHouse will recalculate checksums for parts when file `checksums.txt` is absent. Broken since [#9827](https://github.com/ClickHouse/ClickHouse/issues/9827). [#12545](https://github.com/ClickHouse/ClickHouse/pull/12545) ([alesapin](https://github.com/alesapin)). +* Backported in [#12873](https://github.com/ClickHouse/ClickHouse/issues/12873): Fix error `Output of TreeExecutor is not sorted` for `OPTIMIZE DEDUPLICATE`. Fixes [#11572](https://github.com/ClickHouse/ClickHouse/issues/11572). [#12613](https://github.com/ClickHouse/ClickHouse/pull/12613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#12885](https://github.com/ClickHouse/ClickHouse/issues/12885): Fix possible `Pipeline stuck` error for queries with external sorting. Fixes [#12617](https://github.com/ClickHouse/ClickHouse/issues/12617). [#12618](https://github.com/ClickHouse/ClickHouse/pull/12618) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#12876](https://github.com/ClickHouse/ClickHouse/issues/12876): Better exception message in disk access storage. [#12625](https://github.com/ClickHouse/ClickHouse/pull/12625) ([alesapin](https://github.com/alesapin)). +* Backported in [#12882](https://github.com/ClickHouse/ClickHouse/issues/12882): Exception `There is no supertype...` can be thrown during `ALTER ... UPDATE` in unexpected cases (e.g. when subtracting from UInt64 column). This fixes [#7306](https://github.com/ClickHouse/ClickHouse/issues/7306). This fixes [#4165](https://github.com/ClickHouse/ClickHouse/issues/4165). [#12633](https://github.com/ClickHouse/ClickHouse/pull/12633) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12879](https://github.com/ClickHouse/ClickHouse/issues/12879): Add support for function `if` with `Array(UUID)` arguments. This fixes [#11066](https://github.com/ClickHouse/ClickHouse/issues/11066). [#12648](https://github.com/ClickHouse/ClickHouse/pull/12648) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12972](https://github.com/ClickHouse/ClickHouse/issues/12972): Fix SIGSEGV in StorageKafka when broker is unavailable (and not only). [#12658](https://github.com/ClickHouse/ClickHouse/pull/12658) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#12860](https://github.com/ClickHouse/ClickHouse/issues/12860): fixes [#10572](https://github.com/ClickHouse/ClickHouse/issues/10572) fix bloom filter index with const expression. [#12659](https://github.com/ClickHouse/ClickHouse/pull/12659) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#12869](https://github.com/ClickHouse/ClickHouse/issues/12869): fixes [#12293](https://github.com/ClickHouse/ClickHouse/issues/12293) allow push predicate when subquery contains with clause. [#12663](https://github.com/ClickHouse/ClickHouse/pull/12663) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#12866](https://github.com/ClickHouse/ClickHouse/issues/12866): Fix memory tracking for input_format_parallel_parsing (by attaching thread to group). [#12672](https://github.com/ClickHouse/ClickHouse/pull/12672) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13185](https://github.com/ClickHouse/ClickHouse/issues/13185): Fix performance with large tuples, which are interpreted as functions in `IN` section. The case when user write `WHERE x IN tuple(1, 2, ...)` instead of `WHERE x IN (1, 2, ...)` for some obscure reason. [#12700](https://github.com/ClickHouse/ClickHouse/pull/12700) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#13032](https://github.com/ClickHouse/ClickHouse/issues/13032): Corrected merge_with_ttl_timeout logic which did not work well when expiration affected more than one partition over one time interval. (Authored by @excitoon). [#12982](https://github.com/ClickHouse/ClickHouse/pull/12982) ([Alexander Kazakov](https://github.com/Akazz)). +* Backported in [#13047](https://github.com/ClickHouse/ClickHouse/issues/13047): Fix `Block structure mismatch` error for queries with `UNION` and `JOIN`. Fixes [#12602](https://github.com/ClickHouse/ClickHouse/issues/12602). [#12989](https://github.com/ClickHouse/ClickHouse/pull/12989) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13050](https://github.com/ClickHouse/ClickHouse/issues/13050): Fix crash which was possible for queries with `ORDER BY` tuple and small `LIMIT`. Fixes [#12623](https://github.com/ClickHouse/ClickHouse/issues/12623). [#13009](https://github.com/ClickHouse/ClickHouse/pull/13009) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13144](https://github.com/ClickHouse/ClickHouse/issues/13144): Fix wrong index analysis with functions. It could lead to pruning wrong parts, while reading from `MergeTree` tables. Fixes [#13060](https://github.com/ClickHouse/ClickHouse/issues/13060). Fixes [#12406](https://github.com/ClickHouse/ClickHouse/issues/12406). [#13081](https://github.com/ClickHouse/ClickHouse/pull/13081) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v20.5.5.74-stable.md b/docs/changelogs/v20.5.5.74-stable.md new file mode 100644 index 00000000000..0f2d5f7aab1 --- /dev/null +++ b/docs/changelogs/v20.5.5.74-stable.md @@ -0,0 +1,34 @@ +### ClickHouse release v20.5.5.74-stable FIXME as compared to v20.5.4.40-stable + +#### Improvement +* Backported in [#13920](https://github.com/ClickHouse/ClickHouse/issues/13920): Fix data race in `lgamma` function. This race was caught only in `tsan`, no side effects a really happened. [#13842](https://github.com/ClickHouse/ClickHouse/pull/13842) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Bug Fix +* Backported in [#13300](https://github.com/ClickHouse/ClickHouse/issues/13300): The function `groupArrayMoving*` was not working for distributed queries. It's result was calculated within incorrect data type (without promotion to the largest type). The function `groupArrayMovingAvg` was returning integer number that was inconsistent with the `avg` function. This fixes [#12568](https://github.com/ClickHouse/ClickHouse/issues/12568). [#12622](https://github.com/ClickHouse/ClickHouse/pull/12622) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13030](https://github.com/ClickHouse/ClickHouse/issues/13030): CREATE USER IF NOT EXISTS now doesn't throw exception if the user exists. This fixes [#12507](https://github.com/ClickHouse/ClickHouse/issues/12507). [#12646](https://github.com/ClickHouse/ClickHouse/pull/12646) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#12996](https://github.com/ClickHouse/ClickHouse/issues/12996): Fix optimization `optimize_move_functions_out_of_any=1` in case of `any(func())`. [#12664](https://github.com/ClickHouse/ClickHouse/pull/12664) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#13092](https://github.com/ClickHouse/ClickHouse/issues/13092): Fix CAST(Nullable(String), Enum()). [#12745](https://github.com/ClickHouse/ClickHouse/pull/12745) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#12984](https://github.com/ClickHouse/ClickHouse/issues/12984): Fix columns duplication for range hashed dictionary created from DDL query. This fixes [#10605](https://github.com/ClickHouse/ClickHouse/issues/10605). [#12857](https://github.com/ClickHouse/ClickHouse/pull/12857) ([alesapin](https://github.com/alesapin)). +* Backported in [#13561](https://github.com/ClickHouse/ClickHouse/issues/13561): Fix access to redis dictionary after connection was dropped once. It may happen with `cache` and `direct` dictionary layouts. [#13082](https://github.com/ClickHouse/ClickHouse/pull/13082) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#13508](https://github.com/ClickHouse/ClickHouse/issues/13508): Fix parsing row policies from users.xml when names of databases or tables contain dots. This fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779), [#12527](https://github.com/ClickHouse/ClickHouse/issues/12527). [#13199](https://github.com/ClickHouse/ClickHouse/pull/13199) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#13358](https://github.com/ClickHouse/ClickHouse/issues/13358): AvroConfluent: Skip Kafka tombstone records AvroConfluent: Support skipping broken records ... [#13203](https://github.com/ClickHouse/ClickHouse/pull/13203) ([Andrew Onyshchuk](https://github.com/oandrew)). +* Backported in [#13223](https://github.com/ClickHouse/ClickHouse/issues/13223): Fix DateTime64 conversion functions with constant argument. [#13205](https://github.com/ClickHouse/ClickHouse/pull/13205) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13244](https://github.com/ClickHouse/ClickHouse/issues/13244): Fix assert in `arrayElement` function in case of array elements are Nullable and array subscript is also Nullable. This fixes [#12172](https://github.com/ClickHouse/ClickHouse/issues/12172). [#13224](https://github.com/ClickHouse/ClickHouse/pull/13224) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13268](https://github.com/ClickHouse/ClickHouse/issues/13268): Fix function if with nullable constexpr as cond that is not literal NULL. Fixes [#12463](https://github.com/ClickHouse/ClickHouse/issues/12463). [#13226](https://github.com/ClickHouse/ClickHouse/pull/13226) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13351](https://github.com/ClickHouse/ClickHouse/issues/13351): Return passed number for numbers with MSB set in roundUpToPowerOfTwoOrZero(). [#13234](https://github.com/ClickHouse/ClickHouse/pull/13234) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13299](https://github.com/ClickHouse/ClickHouse/issues/13299): Fix potentially low performance and slightly incorrect result for `uniqExact`, `topK`, `sumDistinct` and similar aggregate functions called on Float types with NaN values. It also triggered assert in debug build. This fixes [#12491](https://github.com/ClickHouse/ClickHouse/issues/12491). [#13254](https://github.com/ClickHouse/ClickHouse/pull/13254) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13315](https://github.com/ClickHouse/ClickHouse/issues/13315): The server may crash if user passed specifically crafted arguments to the function `h3ToChildren`. This fixes [#13275](https://github.com/ClickHouse/ClickHouse/issues/13275). [#13277](https://github.com/ClickHouse/ClickHouse/pull/13277) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13328](https://github.com/ClickHouse/ClickHouse/issues/13328): Fix possible error `Totals having transform was already added to pipeline` in case of a query from delayed replica. [#13290](https://github.com/ClickHouse/ClickHouse/pull/13290) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13609](https://github.com/ClickHouse/ClickHouse/issues/13609): Fix missing or excessive headers in `TSV/CSVWithNames` formats. This fixes [#12504](https://github.com/ClickHouse/ClickHouse/issues/12504). [#13343](https://github.com/ClickHouse/ClickHouse/pull/13343) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13482](https://github.com/ClickHouse/ClickHouse/issues/13482): Fix queries with constant columns and `ORDER BY` prefix of primary key. [#13396](https://github.com/ClickHouse/ClickHouse/pull/13396) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#13488](https://github.com/ClickHouse/ClickHouse/issues/13488): Fix empty output for `Arrow` and `Parquet` formats in case if query return zero rows. It was done because empty output is not valid for this formats. [#13399](https://github.com/ClickHouse/ClickHouse/pull/13399) ([hcz](https://github.com/hczhcz)). +* Backported in [#13569](https://github.com/ClickHouse/ClickHouse/issues/13569): Fix possible race in `StorageMemory`. https://clickhouse-test-reports.s3.yandex.net/0/9cac8a7244063d2092ad25d45502611e18d3749c/stress_test_(thread)/stderr.log Have no idea how to write a test. [#13416](https://github.com/ClickHouse/ClickHouse/pull/13416) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13442](https://github.com/ClickHouse/ClickHouse/issues/13442): Fix `aggregate function any(x) is found inside another aggregate function in query` error with `SET optimize_move_functions_out_of_any = 1` and aliases inside `any()`. [#13419](https://github.com/ClickHouse/ClickHouse/pull/13419) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#13486](https://github.com/ClickHouse/ClickHouse/issues/13486): Fix invalid return type for comparison of tuples with `NULL` elements. Fixes [#12461](https://github.com/ClickHouse/ClickHouse/issues/12461). [#13420](https://github.com/ClickHouse/ClickHouse/pull/13420) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13463](https://github.com/ClickHouse/ClickHouse/issues/13463): Fix error in `parseDateTimeBestEffort` function when unix timestamp was passed as an argument. This fixes [#13362](https://github.com/ClickHouse/ClickHouse/issues/13362). [#13441](https://github.com/ClickHouse/ClickHouse/pull/13441) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13663](https://github.com/ClickHouse/ClickHouse/issues/13663): Concurrent `ALTER ... REPLACE/MOVE PARTITION ...` queries might cause deadlock. It's fixed. [#13626](https://github.com/ClickHouse/ClickHouse/pull/13626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#13717](https://github.com/ClickHouse/ClickHouse/issues/13717): Fix crash in JOIN with StorageMerge and `set enable_optimize_predicate_expression=1`. [#13679](https://github.com/ClickHouse/ClickHouse/pull/13679) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#13701](https://github.com/ClickHouse/ClickHouse/issues/13701): Do not optimize any(arrayJoin()) -> arrayJoin() under optimize_move_functions_out_of_any. [#13681](https://github.com/ClickHouse/ClickHouse/pull/13681) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13732](https://github.com/ClickHouse/ClickHouse/issues/13732): Fix incorrect message in `clickhouse-server.init` while checking user and group. [#13711](https://github.com/ClickHouse/ClickHouse/pull/13711) ([ylchou](https://github.com/ylchou)). +* Backported in [#13903](https://github.com/ClickHouse/ClickHouse/issues/13903): Fix incorrect sorting for `FixedString` columns. Fixes [#13182](https://github.com/ClickHouse/ClickHouse/issues/13182). [#13887](https://github.com/ClickHouse/ClickHouse/pull/13887) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v20.6.1.4066-prestable.md b/docs/changelogs/v20.6.1.4066-prestable.md new file mode 100644 index 00000000000..523709c2be5 --- /dev/null +++ b/docs/changelogs/v20.6.1.4066-prestable.md @@ -0,0 +1,184 @@ +### ClickHouse release v20.6.1.4066-prestable FIXME as compared to v20.5.1.3833-prestable + +#### Backward Incompatible Change +* `clickhouse-local` now uses an unique temporary data directory by default, not the current directory as before. If needed, the data directory can be explicitly specified with the `-- --path` option. [#11931](https://github.com/ClickHouse/ClickHouse/pull/11931) ([Alexander Kuzmenkov](https://github.com/akuzm)). + +#### New Feature +* Add `Alter table drop replica replica_name` support. This fixes [#7080](https://github.com/ClickHouse/ClickHouse/issues/7080). [#10679](https://github.com/ClickHouse/ClickHouse/pull/10679) ([sundyli](https://github.com/sundy-li)). +* Added new in-memory format of parts in `MergeTree`-family tables, which stores data in memory. Parts are written on disk at first merge. Part will be created in in-memory format if its size in rows or bytes is below thresholds `min_rows_for_compact_part` and `min_bytes_for_compact_part`. Also optional support of Write-Ahead-Log is available, which is enabled by default and is controlled by setting `in_memory_parts_enable_wal`. [#10697](https://github.com/ClickHouse/ClickHouse/pull/10697) ([Anton Popov](https://github.com/CurtizJ)). +* Add -Distinct combinator for aggregate functions. [#10930](https://github.com/ClickHouse/ClickHouse/pull/10930) ([Sofia Antipushina](https://github.com/Sonichka1311)). +* Support table engine mongo(host:port, database, collection, user, password). [#10931](https://github.com/ClickHouse/ClickHouse/pull/10931) ([ageraab](https://github.com/ageraab)). +* Add storage RabbitMQ. [#11069](https://github.com/ClickHouse/ClickHouse/pull/11069) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Opt-in settings to send crash reports to the ClickHouse core team via [Sentry](https://sentry.io). [#11300](https://github.com/ClickHouse/ClickHouse/pull/11300) ([Ivan Blinkov](https://github.com/blinkov)). +* Add ORCBlockOutputFormat. [#11662](https://github.com/ClickHouse/ClickHouse/pull/11662) ([Kruglov Pavel](https://github.com/Avogar)). +* `max_thread_pool_size` config for changing the maximum number of Threads in Global Thread Pool. [#11668](https://github.com/ClickHouse/ClickHouse/pull/11668) ([Bharat Nallan](https://github.com/bharatnc)). +* Initial implementation of `EXPLAIN` query. Syntax: `EXPLAIN SELECT ...`. This fixes [#1118](https://github.com/ClickHouse/ClickHouse/issues/1118). [#11873](https://github.com/ClickHouse/ClickHouse/pull/11873) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Switched paths in S3 metadata to relative which allows to handle S3 blobs more easily. [#11892](https://github.com/ClickHouse/ClickHouse/pull/11892) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Avro UUID input/output support. [#11954](https://github.com/ClickHouse/ClickHouse/pull/11954) ([Andrew Onyshchuk](https://github.com/oandrew)). +* Added read-only MongoDB table engine. Allows reading flat (primitive types, not nested) fields. [#11983](https://github.com/ClickHouse/ClickHouse/pull/11983) ([alesapin](https://github.com/alesapin)). +* Add setting to fields not found in Avro schema. [#12007](https://github.com/ClickHouse/ClickHouse/pull/12007) ([Andrew Onyshchuk](https://github.com/oandrew)). +* add function `parseDateTimeBestEffortUS`. [#12028](https://github.com/ClickHouse/ClickHouse/pull/12028) ([flynn](https://github.com/ucasfl)). +* #4006 Support ALTER TABLE ... [ADD|MODIFY] COLUMN ... FIRST. [#12073](https://github.com/ClickHouse/ClickHouse/pull/12073) ([Winter Zhang](https://github.com/zhang2014)). +* Add a function initializedAggregation to initialize an aggregation based on a single value. [#12109](https://github.com/ClickHouse/ClickHouse/pull/12109) ([Guillaume Tassery](https://github.com/YiuRULE)). +* Support RIGHT and FULL JOIN with `set join_algorithm=partial_merge`. Only ALL strictness is supported (ANY, SEMI, ANTI, ASOF are not). [#12118](https://github.com/ClickHouse/ClickHouse/pull/12118) ([Artem Zuikov](https://github.com/4ertus2)). +* Implementation of PostgreSQL-like ILIKE operator for [#11710](https://github.com/ClickHouse/ClickHouse/issues/11710). [#12125](https://github.com/ClickHouse/ClickHouse/pull/12125) ([Mike Kot](https://github.com/myrrc)). +* Allow Nullable types as keys in MergeTree tables. [#5319](https://github.com/ClickHouse/ClickHouse/issues/5319). [#12433](https://github.com/ClickHouse/ClickHouse/pull/12433) ([Amos Bird](https://github.com/amosbird)). + +#### Performance Improvement +* Allow to use direct_io and mmap_io for secondary indices if the settings `min_bytes_to_use_direct_io` or `min_bytes_to_use_mmap_io` are configured. [#11955](https://github.com/ClickHouse/ClickHouse/pull/11955) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix estimation of the number of marks while reading from MergeTree. This is needed to correctly handle the settings `merge_tree_max_rows_to_use_cache`, `merge_tree_max_bytes_to_use_cache`, `merge_tree_min_rows_for_concurrent_read`, `merge_tree_min_bytes_for_concurrent_read`, `merge_tree_min_rows_for_seek`, `merge_tree_min_bytes_for_seek`. Now settings `min_bytes_to_use_mmap_io` also applied to read index and compact parts in MergeTree table engines family. [#11970](https://github.com/ClickHouse/ClickHouse/pull/11970) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix "[#10574](https://github.com/ClickHouse/ClickHouse/issues/10574) Index not used for IN operator with literals", performance regression introduced around v19.3. [#12062](https://github.com/ClickHouse/ClickHouse/pull/12062) ([nvartolomei](https://github.com/nvartolomei)). +* Remove injective functions inside `uniq*()` if `set optimize_injective_functions_inside_uniq=1`. [#12337](https://github.com/ClickHouse/ClickHouse/pull/12337) ([Artem Zuikov](https://github.com/4ertus2)). +* Add order by optimisation that rewrites `ORDER BY x, f(x)` with `ORDER by x` if `set optimize_redundant_functions_in_order_by = 1`. [#12404](https://github.com/ClickHouse/ClickHouse/pull/12404) ([Artem Zuikov](https://github.com/4ertus2)). +* Lower memory usage for some operations up to 2 times. [#12424](https://github.com/ClickHouse/ClickHouse/pull/12424) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Add number of errors to ignore while choosing replicas (`distributed_replica_error_ignore`). [#11669](https://github.com/ClickHouse/ClickHouse/pull/11669) ([Azat Khuzhin](https://github.com/azat)). +* Improved performace of 'ORDER BY' and 'GROUP BY' by prefix of sorting key. [#11696](https://github.com/ClickHouse/ClickHouse/pull/11696) ([Anton Popov](https://github.com/CurtizJ)). +* - Add `optimize_skip_unused_shards_nesting` (allows control nesting level for shards skipping optimization) - Add `force_skip_optimize_shards_nesting` (allows control nesting level for checking was shards skipped or not) - Deprecate `force_optimize_skip_unused_shards_no_nested` (`force_skip_optimize_shards_nesting` should be used instead) - Disable `optimize_skip_unused_shards` if sharding_key has non-deterministic func (i.e. `rand()`, note that this does not changes anything for INSERT side). [#11715](https://github.com/ClickHouse/ClickHouse/pull/11715) ([Azat Khuzhin](https://github.com/azat)). +* Multiversion metadata for storages without structure locks. [#11745](https://github.com/ClickHouse/ClickHouse/pull/11745) ([alesapin](https://github.com/alesapin)). +* Slightly relax the validation of ODBC connection string. If the hostname or username contains only word characters along with `.` and `-`, don't put it into curly braces. It is needed, because some ODBC drivers (e.g. PostgreSQL) don't understand when hostname is enclosed in curly braces. [#11845](https://github.com/ClickHouse/ClickHouse/pull/11845) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support parse UUID without separator(separators are always removed in most implementations, this is helpful for users to write data). [#11856](https://github.com/ClickHouse/ClickHouse/pull/11856) ([Winter Zhang](https://github.com/zhang2014)). +* Support `SIGNED` and `UNSIGNED` modifiers of standard integer types (`BIGINT`, `INT`, ...) for compatibility with MySQL. [#11858](https://github.com/ClickHouse/ClickHouse/pull/11858) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Allow to use `sumWithOverflow` as `SimpleAggregateFunction`. Closes [#8053](https://github.com/ClickHouse/ClickHouse/issues/8053). [#11865](https://github.com/ClickHouse/ClickHouse/pull/11865) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add FixedString support in Hashing functions. [#11878](https://github.com/ClickHouse/ClickHouse/pull/11878) ([flynn](https://github.com/ucasfl)). +* Rewrite code for `optimize_arithmetic_operations_in_aggregate_functions` optimisation. [#11899](https://github.com/ClickHouse/ClickHouse/pull/11899) ([Artem Zuikov](https://github.com/4ertus2)). +* Improve path concatenation and fix double slashed paths using std::filesystem::path instead of std::string in `DatabaseOrdinary.cpp`. [#11900](https://github.com/ClickHouse/ClickHouse/pull/11900) ([Bharat Nallan](https://github.com/bharatnc)). +* Deprecate the old regular style and use the new globalVariable method [#11832](https://github.com/ClickHouse/ClickHouse/issues/11832). [#11901](https://github.com/ClickHouse/ClickHouse/pull/11901) ([BohuTANG](https://github.com/BohuTANG)). +* related to [issue 9797](https://github.com/ClickHouse/ClickHouse/issues/9797). [#11923](https://github.com/ClickHouse/ClickHouse/pull/11923) ([flynn](https://github.com/ucasfl)). +* `system.tables` now considers column capacities for Memory and Buffer table engines, which is better approximation for resident memory size. [#11935](https://github.com/ClickHouse/ClickHouse/pull/11935) ([Max Akhmedov](https://github.com/zlobober)). +* Add CPU frequencies to system.asynchronous_metrics. Make the metric collection period configurable. [#11972](https://github.com/ClickHouse/ClickHouse/pull/11972) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow to perform "metadata-only" ALTER of partition key such as adding more elements to Enum data type. This fixes [#7513](https://github.com/ClickHouse/ClickHouse/issues/7513). [#11973](https://github.com/ClickHouse/ClickHouse/pull/11973) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add replica priority for load_balancing (for manual prioritization of the load balancing). [#11995](https://github.com/ClickHouse/ClickHouse/pull/11995) ([Azat Khuzhin](https://github.com/azat)). +* Support MySQL engine reading Enums type [#3985](https://github.com/ClickHouse/ClickHouse/issues/3985). [#11996](https://github.com/ClickHouse/ClickHouse/pull/11996) ([BohuTANG](https://github.com/BohuTANG)). +* Implemented single part uploads for DiskS3. [#12026](https://github.com/ClickHouse/ClickHouse/pull/12026) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Moved useless S3 logging to TRACE level. [#12067](https://github.com/ClickHouse/ClickHouse/pull/12067) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Improves `REVOKE` command: now it requires grant/admin option for only access which will be revoked. For example, to execute `REVOKE ALL ON *.* FROM user1` now it doesn't require to have full access rights granted with grant option. Added command `REVOKE ALL FROM user1` - it revokes all granted roles from `user1`. [#12083](https://github.com/ClickHouse/ClickHouse/pull/12083) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add 'type' column in system.disks. [#12115](https://github.com/ClickHouse/ClickHouse/pull/12115) ([ianton-ru](https://github.com/ianton-ru)). +* Added support for `%g` (two digit ISO year) and `%G` (four digit ISO year) substitutions in `formatDateTime` function. [#12136](https://github.com/ClickHouse/ClickHouse/pull/12136) ([vivarum](https://github.com/vivarum)). +* Add `KILL QUERY [connection_id]` for the MySQL client/driver to cancel the long query, issue [#12038](https://github.com/ClickHouse/ClickHouse/issues/12038). [#12152](https://github.com/ClickHouse/ClickHouse/pull/12152) ([BohuTANG](https://github.com/BohuTANG)). +* 1. Support MySQL 'SELECT DATABASE()' [#9336](https://github.com/ClickHouse/ClickHouse/issues/9336) 2. Add MySQL replacement query integration test. [#12314](https://github.com/ClickHouse/ClickHouse/pull/12314) ([BohuTANG](https://github.com/BohuTANG)). +* This setting allows to chose charset for printing grids (either utf8 or ascii). [#12372](https://github.com/ClickHouse/ClickHouse/pull/12372) ([Sabyanin Maxim](https://github.com/s-mx)). +* Write the detail exception message to the client instead of 'MySQL server has gone away'. [#12383](https://github.com/ClickHouse/ClickHouse/pull/12383) ([BohuTANG](https://github.com/BohuTANG)). +* lifetime_rows/lifetime_bytes for Buffer engine. [#12421](https://github.com/ClickHouse/ClickHouse/pull/12421) ([Azat Khuzhin](https://github.com/azat)). +* Use correct default secure port for clickhouse-benchmark with `--secure` argument. This fixes [#11044](https://github.com/ClickHouse/ClickHouse/issues/11044). [#12440](https://github.com/ClickHouse/ClickHouse/pull/12440) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Fix unexpected behaviour of queries like `SELECT *, xyz.*` which were success while an error expected. [#11753](https://github.com/ClickHouse/ClickHouse/pull/11753) ([hexiaoting](https://github.com/hexiaoting)). +* Fix wrong result for `if()` with NULLs in condition. [#11807](https://github.com/ClickHouse/ClickHouse/pull/11807) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix memory accounting via HTTP interface (can be significant with `wait_end_of_query=1`). [#11840](https://github.com/ClickHouse/ClickHouse/pull/11840) ([Azat Khuzhin](https://github.com/azat)). +* Fix rare crash caused by using `Nullable` column in prewhere condition. Continuation of [#11608](https://github.com/ClickHouse/ClickHouse/issues/11608). [#11869](https://github.com/ClickHouse/ClickHouse/pull/11869) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix potential floating point exception when parsing DateTime64. This fixes [#11374](https://github.com/ClickHouse/ClickHouse/issues/11374). [#11875](https://github.com/ClickHouse/ClickHouse/pull/11875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed bug with no moves when changing storage policy from default one. [#11893](https://github.com/ClickHouse/ClickHouse/pull/11893) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix rare crash caused by using Nullable column in prewhere condition. Continuation of [#11869](https://github.com/ClickHouse/ClickHouse/issues/11869). [#11895](https://github.com/ClickHouse/ClickHouse/pull/11895) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Keep aliases for substitutions in query (parametrized queries). This fixes [#11914](https://github.com/ClickHouse/ClickHouse/issues/11914). [#11916](https://github.com/ClickHouse/ClickHouse/pull/11916) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix unitialized memory in partitions serialization. [#11919](https://github.com/ClickHouse/ClickHouse/pull/11919) ([alesapin](https://github.com/alesapin)). +* Use the correct current database for checking access rights after statement `USE database`. [#11920](https://github.com/ClickHouse/ClickHouse/pull/11920) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed `Query parameter was not set` in `Values` format. Fixes [#11918](https://github.com/ClickHouse/ClickHouse/issues/11918). [#11936](https://github.com/ClickHouse/ClickHouse/pull/11936) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix race condition in extractAllGroups* functions. [#11949](https://github.com/ClickHouse/ClickHouse/pull/11949) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make mmap IO work again (experimental). Continuation of [#8520](https://github.com/ClickHouse/ClickHouse/issues/8520). [#11953](https://github.com/ClickHouse/ClickHouse/pull/11953) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong setting name in log message at server startup. [#11997](https://github.com/ClickHouse/ClickHouse/pull/11997) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Partial revokes work correctly in complex cases as well, for example. [#12002](https://github.com/ClickHouse/ClickHouse/pull/12002) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix potential floating point exception. This closes [#11378](https://github.com/ClickHouse/ClickHouse/issues/11378). [#12005](https://github.com/ClickHouse/ClickHouse/pull/12005) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid returning wrong number of geohashes in function `geoHashesInBox` due to accumulation of floating point error. This fixes [#11369](https://github.com/ClickHouse/ClickHouse/issues/11369). [#12006](https://github.com/ClickHouse/ClickHouse/pull/12006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potential array size overflow in generateRandom that may lead to crash. This fixes [#11371](https://github.com/ClickHouse/ClickHouse/issues/11371). [#12013](https://github.com/ClickHouse/ClickHouse/pull/12013) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix calculation of access rights when allow_ddl=0. [#12015](https://github.com/ClickHouse/ClickHouse/pull/12015) ([Vitaly Baranov](https://github.com/vitlibar)). +* When adding floating point number of intervals to date/datetime, the result may be calculated incorrectly. This fixes [#11377](https://github.com/ClickHouse/ClickHouse/issues/11377). [#12018](https://github.com/ClickHouse/ClickHouse/pull/12018) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* A query with function `neighbor` as the only returned expression may return empty result if the function is called with offset `-9223372036854775808`. This fixes [#11367](https://github.com/ClickHouse/ClickHouse/issues/11367). [#12019](https://github.com/ClickHouse/ClickHouse/pull/12019) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not mention in changelog, because the bug did not come to release. Fix potential crash when doing ORDER BY multiple columns with specified COLLATE on one of the column when this column is constant. This fixes [#11379](https://github.com/ClickHouse/ClickHouse/issues/11379). The bug was introduced in [#11006](https://github.com/ClickHouse/ClickHouse/issues/11006) in version 20.5. [#12020](https://github.com/ClickHouse/ClickHouse/pull/12020) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong result and potential crash when invoking function `if` with arguments of type `FixedString` with different sizes. This fixes [#11362](https://github.com/ClickHouse/ClickHouse/issues/11362). [#12021](https://github.com/ClickHouse/ClickHouse/pull/12021) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix calculation of access rights when allow_introspection_functions=0. [#12031](https://github.com/ClickHouse/ClickHouse/pull/12031) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix crash in JOIN with LowCardinality type with `join_algorithm=partial_merge`. [#12035](https://github.com/ClickHouse/ClickHouse/pull/12035) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix incorrect comparison of tuples with `Nullable` columns. Fixes [#11985](https://github.com/ClickHouse/ClickHouse/issues/11985). [#12039](https://github.com/ClickHouse/ClickHouse/pull/12039) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix constraints check if constraint is a constant expression. This fixes [#11360](https://github.com/ClickHouse/ClickHouse/issues/11360). [#12042](https://github.com/ClickHouse/ClickHouse/pull/12042) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `topK` aggregate function return Enum for Enum types. This fixes [#3740](https://github.com/ClickHouse/ClickHouse/issues/3740). [#12043](https://github.com/ClickHouse/ClickHouse/pull/12043) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Parse tables metadata in parallel when loading database. This fixes slow server startup when there are large number of tables. [#12045](https://github.com/ClickHouse/ClickHouse/pull/12045) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error `Cannot capture column` for higher-order functions with `Tuple(LowCardinality)` argument. Fixes [#9766](https://github.com/ClickHouse/ClickHouse/issues/9766). [#12055](https://github.com/ClickHouse/ClickHouse/pull/12055) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix error `Expected single dictionary argument for function` for function `defaultValueOfArgumentType` with `LowCardinality` type. Fixes [#11808](https://github.com/ClickHouse/ClickHouse/issues/11808). [#12056](https://github.com/ClickHouse/ClickHouse/pull/12056) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible crash while using wrong type for `PREWHERE`. Fixes [#12053](https://github.com/ClickHouse/ClickHouse/issues/12053), [#12060](https://github.com/ClickHouse/ClickHouse/issues/12060). [#12060](https://github.com/ClickHouse/ClickHouse/pull/12060) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix SIGSEGV in StorageKafka on DROP TABLE. [#12075](https://github.com/ClickHouse/ClickHouse/pull/12075) ([Azat Khuzhin](https://github.com/azat)). +* Fix unnecessary limiting the number of threads for selects from `VIEW`. Fixes [#11937](https://github.com/ClickHouse/ClickHouse/issues/11937). [#12085](https://github.com/ClickHouse/ClickHouse/pull/12085) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix empty `result_rows` and `result_bytes` metrics in `system.quey_log` for selects. Fixes [#11595](https://github.com/ClickHouse/ClickHouse/issues/11595). [#12089](https://github.com/ClickHouse/ClickHouse/pull/12089) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix segfault with `-StateResample` combinators. [#12092](https://github.com/ClickHouse/ClickHouse/pull/12092) ([Anton Popov](https://github.com/CurtizJ)). +* Fix performance for selects with `UNION` caused by wrong limit for the total number of threads. Fixes [#12030](https://github.com/ClickHouse/ClickHouse/issues/12030). [#12103](https://github.com/ClickHouse/ClickHouse/pull/12103) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Format `Parquet` now properly works with `LowCardinality` and `LowCardinality(Nullable)` types. Fixes [#12086](https://github.com/ClickHouse/ClickHouse/issues/12086), [#8406](https://github.com/ClickHouse/ClickHouse/issues/8406). [#12108](https://github.com/ClickHouse/ClickHouse/pull/12108) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix handling dependency of table with ENGINE=Dictionary on dictionary. This fixes [#10994](https://github.com/ClickHouse/ClickHouse/issues/10994). This fixes [#10397](https://github.com/ClickHouse/ClickHouse/issues/10397). [#12116](https://github.com/ClickHouse/ClickHouse/pull/12116) ([Vitaly Baranov](https://github.com/vitlibar)). +* Avoid "There is no query" exception for materialized views with joins or with subqueries attached to system logs (system.query_log, metric_log, etc) or to engine=Buffer underlying table. [#12120](https://github.com/ClickHouse/ClickHouse/pull/12120) ([filimonov](https://github.com/filimonov)). +* Fix bug which leads to incorrect table metadata in ZooKeepeer for ReplicatedVersionedCollapsingMergeTree tables. Fixes [#12093](https://github.com/ClickHouse/ClickHouse/issues/12093). [#12121](https://github.com/ClickHouse/ClickHouse/pull/12121) ([alesapin](https://github.com/alesapin)). +* Normalize "pid" file handling. In previous versions the server may refuse to start if it was killed without proper shutdown and if there is another process that has the same pid as previously runned server. Also pid file may be removed in unsuccessful server startup even if there is another server running. This fixes [#3501](https://github.com/ClickHouse/ClickHouse/issues/3501). [#12133](https://github.com/ClickHouse/ClickHouse/pull/12133) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potential infinite loop in `greatCircleDistance`, `geoDistance`. This fixes [#12117](https://github.com/ClickHouse/ClickHouse/issues/12117). [#12137](https://github.com/ClickHouse/ClickHouse/pull/12137) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potential overflow in integer division. This fixes [#12119](https://github.com/ClickHouse/ClickHouse/issues/12119). [#12140](https://github.com/ClickHouse/ClickHouse/pull/12140) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad code in redundant ORDER BY optimization. The bug was introduced in [#10067](https://github.com/ClickHouse/ClickHouse/issues/10067). [#12148](https://github.com/ClickHouse/ClickHouse/pull/12148) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix transform of query to send to external DBMS (e.g. MySQL, ODBC) in presense of aliases. This fixes [#12032](https://github.com/ClickHouse/ClickHouse/issues/12032). [#12151](https://github.com/ClickHouse/ClickHouse/pull/12151) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong logic in ALTER DELETE that leads to deleting of records when condition evaluates to NULL. This fixes [#9088](https://github.com/ClickHouse/ClickHouse/issues/9088). This closes [#12106](https://github.com/ClickHouse/ClickHouse/issues/12106). [#12153](https://github.com/ClickHouse/ClickHouse/pull/12153) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't split the dictionary source's table name into schema and table name itself if ODBC connection doesn't support schema. [#12165](https://github.com/ClickHouse/ClickHouse/pull/12165) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed the behaviour when `SummingMergeTree` engine sums up columns from partition key. Added an exception in case of explicit definition of columns to sum which intersects with partition key columns. This fixes [#7867](https://github.com/ClickHouse/ClickHouse/issues/7867). [#12173](https://github.com/ClickHouse/ClickHouse/pull/12173) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix dictGet arguments check during GROUP BY injective functions elimination. [#12179](https://github.com/ClickHouse/ClickHouse/pull/12179) ([Azat Khuzhin](https://github.com/azat)). +* Cap max_memory_usage* limits to the process resident memory. [#12182](https://github.com/ClickHouse/ClickHouse/pull/12182) ([Azat Khuzhin](https://github.com/azat)). +* Fixed logical functions for UInt8 values when they are not equal to 0 or 1. [#12196](https://github.com/ClickHouse/ClickHouse/pull/12196) ([Alexander Kazakov](https://github.com/Akazz)). +* Fixed the behaviour when during multiple sequential inserts in `StorageFile` header for some special types was written more than once. This fixed [#6155](https://github.com/ClickHouse/ClickHouse/issues/6155). [#12197](https://github.com/ClickHouse/ClickHouse/pull/12197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed behaviour on reaching redirect limit in request to S3 storage. [#12256](https://github.com/ClickHouse/ClickHouse/pull/12256) ([ianton-ru](https://github.com/ianton-ru)). +* Not for changelog. Cherry-pick after [#12196](https://github.com/ClickHouse/ClickHouse/issues/12196). [#12271](https://github.com/ClickHouse/ClickHouse/pull/12271) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Implement conversions to the common type for LowCardinality types. This allows to execute UNION ALL of tables with columns of LowCardinality and other columns. This fixes [#8212](https://github.com/ClickHouse/ClickHouse/issues/8212). This fixes [#4342](https://github.com/ClickHouse/ClickHouse/issues/4342). [#12275](https://github.com/ClickHouse/ClickHouse/pull/12275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The function `arrayFill` worked incorrectly for empty arrays that may lead to crash. This fixes [#12263](https://github.com/ClickHouse/ClickHouse/issues/12263). [#12279](https://github.com/ClickHouse/ClickHouse/pull/12279) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Show error after TrieDictionary failed to load. [#12290](https://github.com/ClickHouse/ClickHouse/pull/12290) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix typo in setting name. [#12292](https://github.com/ClickHouse/ClickHouse/pull/12292) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Some threads might randomly hang for a few seconds during DNS cache updating. It's fixed. [#12296](https://github.com/ClickHouse/ClickHouse/pull/12296) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix TTL after renaming column, on which depends TTL expression. [#12304](https://github.com/ClickHouse/ClickHouse/pull/12304) ([Anton Popov](https://github.com/CurtizJ)). +* Avoid "bad cast" exception when there is an expression that filters data by virtual columns (like `_table` in `Merge` tables) or by "index" columns in system tables such as filtering by database name when querying from `system.tables`, and this expression returns `Nullable` type. This fixes [#12166](https://github.com/ClickHouse/ClickHouse/issues/12166). [#12305](https://github.com/ClickHouse/ClickHouse/pull/12305) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix order of columns in `WITH FILL` modifier. Previously order of columns of `ORDER BY` statement wasn't respected. [#12306](https://github.com/ClickHouse/ClickHouse/pull/12306) ([Anton Popov](https://github.com/CurtizJ)). +* When using codec `Delta` or `DoubleDelta` with non fixed width types, exception with code `LOGICAL_ERROR` was returned instead of exception with code `BAD_ARGUMENTS` (we ensure that exceptions with code logical error never happen). This fixes [#12110](https://github.com/ClickHouse/ClickHouse/issues/12110). [#12308](https://github.com/ClickHouse/ClickHouse/pull/12308) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix very rare race condition in ReplicatedMergeTreeQueue. [#12315](https://github.com/ClickHouse/ClickHouse/pull/12315) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error message and exit codes for `ALTER RENAME COLUMN` queries, when `RENAME` is not allowed. Fixes [#12301](https://github.com/ClickHouse/ClickHouse/issues/12301) and [#12303](https://github.com/ClickHouse/ClickHouse/issues/12303). [#12335](https://github.com/ClickHouse/ClickHouse/pull/12335) ([alesapin](https://github.com/alesapin)). +* Fix TOTALS/ROLLUP/CUBE for aggregate functions with `-State` and `Nullable` arguments. This fixes [#12163](https://github.com/ClickHouse/ClickHouse/issues/12163). [#12376](https://github.com/ClickHouse/ClickHouse/pull/12376) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to CLEAR column even if there are depending DEFAULT expressions. This fixes [#12333](https://github.com/ClickHouse/ClickHouse/issues/12333). [#12378](https://github.com/ClickHouse/ClickHouse/pull/12378) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid exception when negative or floating point constant is used in WHERE condition for indexed tables. This fixes [#11905](https://github.com/ClickHouse/ClickHouse/issues/11905). [#12384](https://github.com/ClickHouse/ClickHouse/pull/12384) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Additional check for arguments of bloom filter index. This fixes [#11408](https://github.com/ClickHouse/ClickHouse/issues/11408). [#12388](https://github.com/ClickHouse/ClickHouse/pull/12388) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Reverts change introduced in [#11079](https://github.com/ClickHouse/ClickHouse/issues/11079) to resolve [#12098](https://github.com/ClickHouse/ClickHouse/issues/12098). [#12397](https://github.com/ClickHouse/ClickHouse/pull/12397) ([Mike Kot](https://github.com/myrrc)). +* Fixed possible segfault if StorageMerge. Closes [#12054](https://github.com/ClickHouse/ClickHouse/issues/12054). [#12401](https://github.com/ClickHouse/ClickHouse/pull/12401) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix overflow when very large LIMIT or OFFSET is specified. This fixes [#10470](https://github.com/ClickHouse/ClickHouse/issues/10470). This fixes [#11372](https://github.com/ClickHouse/ClickHouse/issues/11372). [#12427](https://github.com/ClickHouse/ClickHouse/pull/12427) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in base64 if tests were run on server with AVX-512. This fixes [#12318](https://github.com/ClickHouse/ClickHouse/issues/12318). Author: @qoega. [#12441](https://github.com/ClickHouse/ClickHouse/pull/12441) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Build/Testing/Packaging Improvement +* Add simple GitHub hook script for the serverless environment. [#11605](https://github.com/ClickHouse/ClickHouse/pull/11605) ([alesapin](https://github.com/alesapin)). +* Send logs to client on fatal errors if possible. This will make test results more readable. [#11826](https://github.com/ClickHouse/ClickHouse/pull/11826) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow ClickHouse to run on Android. [#11894](https://github.com/ClickHouse/ClickHouse/pull/11894) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Clean up unused header files from `Obfuscator.cpp` and `DatabaseAtomic.cpp`. [#11922](https://github.com/ClickHouse/ClickHouse/pull/11922) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix some typos in code. [#12003](https://github.com/ClickHouse/ClickHouse/pull/12003) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Scripts for automated scheduled backporting based on PR labels. [#12029](https://github.com/ClickHouse/ClickHouse/pull/12029) ([Ivan](https://github.com/abyss7)). +* Add new type of tests based on Testflows framework. [#12090](https://github.com/ClickHouse/ClickHouse/pull/12090) ([vzakaznikov](https://github.com/vzakaznikov)). +* Install `ca-certificates` before the first `apt-get update` in Dockerfile. [#12095](https://github.com/ClickHouse/ClickHouse/pull/12095) ([Ivan Blinkov](https://github.com/blinkov)). +* Daily check by [GitHub CodeQL](https://securitylab.github.com/tools/codeql) security analysis tool that looks for [CWE](https://cwe.mitre.org/). [#12101](https://github.com/ClickHouse/ClickHouse/pull/12101) ([Ivan Blinkov](https://github.com/blinkov)). +* Regular check by [Anchore Container Analysis](https://docs.anchore.com) security analysis tool that looks for [CVE](https://cve.mitre.org/) in `clickhouse-server` Docker image. Also confirms that `Dockerfile` is buildable. Runs daily on `master` and on pull-requests to `Dockerfile`. [#12102](https://github.com/ClickHouse/ClickHouse/pull/12102) ([Ivan Blinkov](https://github.com/blinkov)). +* Add `UNBUNDLED` flag to `system.build_options` table. Move skip lists for `clickhouse-test` to clickhouse repo. [#12107](https://github.com/ClickHouse/ClickHouse/pull/12107) ([alesapin](https://github.com/alesapin)). +* Implement AST-based query fuzzing mode for clickhouse-client. See [this label](https://github.com/ClickHouse/ClickHouse/issues?q=label%3Afuzz+is%3Aissue) for the list of issues we recently found by fuzzing. Most of them were found by this tool, and a couple by SQLancer and `00746_sql_fuzzy.pl`. [#12111](https://github.com/ClickHouse/ClickHouse/pull/12111) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Perform an upgrade of system packages in the `clickhouse-server` docker image. [#12124](https://github.com/ClickHouse/ClickHouse/pull/12124) ([Ivan Blinkov](https://github.com/blinkov)). +* Added a showcase of the minimal Docker image without using any Linux distribution. [#12126](https://github.com/ClickHouse/ClickHouse/pull/12126) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor CMake fixes for UNBUNDLED build. [#12131](https://github.com/ClickHouse/ClickHouse/pull/12131) ([Matwey V. Kornilov](https://github.com/matwey)). +* Missed `` is required for `std::atomic<>`. [#12134](https://github.com/ClickHouse/ClickHouse/pull/12134) ([Matwey V. Kornilov](https://github.com/matwey)). +* Fix warnings from CodeQL. `CodeQL` is another static analyzer that we will use along with `clang-tidy` and `PVS-Studio` that we use already. [#12138](https://github.com/ClickHouse/ClickHouse/pull/12138) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Record additional detail on Dockerfile scan reports. [#12159](https://github.com/ClickHouse/ClickHouse/pull/12159) ([Ivan Blinkov](https://github.com/blinkov)). +* Place common docker compose files to integration docker container. [#12168](https://github.com/ClickHouse/ClickHouse/pull/12168) ([Ilya Yatsishin](https://github.com/qoega)). +* Remove verbosity from the binary builds. [#12174](https://github.com/ClickHouse/ClickHouse/pull/12174) ([alesapin](https://github.com/alesapin)). +* Remove strange file creation during build in `orc`. [#12258](https://github.com/ClickHouse/ClickHouse/pull/12258) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Now functional and stress tests will be able to run with old version of `clickhouse-test` script. [#12287](https://github.com/ClickHouse/ClickHouse/pull/12287) ([alesapin](https://github.com/alesapin)). +* Log sanitizer trap messages from separate thread. This will prevent possible deadlock under thread sanitizer. [#12313](https://github.com/ClickHouse/ClickHouse/pull/12313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added S3 HTTPS integration test. [#12412](https://github.com/ClickHouse/ClickHouse/pull/12412) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix UBSan report in HDFS library. This closes [#12330](https://github.com/ClickHouse/ClickHouse/issues/12330). [#12453](https://github.com/ClickHouse/ClickHouse/pull/12453) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Other +* Update word break characters to match readline default - all non-alphanumeric characters. ... [#11975](https://github.com/ClickHouse/ClickHouse/pull/11975) ([Andrew Onyshchuk](https://github.com/oandrew)). + +#### NO CL CATEGORY + +* * Not for changelog. [#12265](https://github.com/ClickHouse/ClickHouse/pull/12265) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ... [#12431](https://github.com/ClickHouse/ClickHouse/pull/12431) ([Tom Bombadil](https://github.com/ithangzhou)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Bump certifi from 2020.4.5.2 to 2020.6.20 in /docs/tools/translate'. [#11853](https://github.com/ClickHouse/ClickHouse/pull/11853) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Merging [#10679](https://github.com/ClickHouse/ClickHouse/issues/10679)'. [#11896](https://github.com/ClickHouse/ClickHouse/pull/11896) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "[experiment] maybe fix warnings in integration tests"'. [#12011](https://github.com/ClickHouse/ClickHouse/pull/12011) ([alesapin](https://github.com/alesapin)). +* NO CL ENTRY: 'Bump idna from 2.9 to 2.10 in /docs/tools'. [#12024](https://github.com/ClickHouse/ClickHouse/pull/12024) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump python-slugify from 1.2.6 to 4.0.1 in /docs/tools'. [#12049](https://github.com/ClickHouse/ClickHouse/pull/12049) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). + diff --git a/docs/changelogs/v20.6.10.2-stable.md b/docs/changelogs/v20.6.10.2-stable.md new file mode 100644 index 00000000000..489fd86ecf5 --- /dev/null +++ b/docs/changelogs/v20.6.10.2-stable.md @@ -0,0 +1,2 @@ +### ClickHouse release v20.6.10.2-stable FIXME as compared to v20.6.9.1-stable + diff --git a/docs/changelogs/v20.6.11.1-stable.md b/docs/changelogs/v20.6.11.1-stable.md new file mode 100644 index 00000000000..d00259e2248 --- /dev/null +++ b/docs/changelogs/v20.6.11.1-stable.md @@ -0,0 +1,2 @@ +### ClickHouse release v20.6.11.1-stable FIXME as compared to v20.6.10.2-stable + diff --git a/docs/changelogs/v20.6.2.15-prestable.md b/docs/changelogs/v20.6.2.15-prestable.md new file mode 100644 index 00000000000..c1ea3c7a665 --- /dev/null +++ b/docs/changelogs/v20.6.2.15-prestable.md @@ -0,0 +1,204 @@ +### ClickHouse release v20.6.2.15-prestable FIXME as compared to v20.5.1.3833-prestable + +#### Backward Incompatible Change +* `clickhouse-local` now uses an unique temporary data directory by default, not the current directory as before. If needed, the data directory can be explicitly specified with the `-- --path` option. [#11931](https://github.com/ClickHouse/ClickHouse/pull/11931) ([Alexander Kuzmenkov](https://github.com/akuzm)). + +#### New Feature +* Add `Alter table drop replica replica_name` support. This fixes [#7080](https://github.com/ClickHouse/ClickHouse/issues/7080). [#10679](https://github.com/ClickHouse/ClickHouse/pull/10679) ([sundyli](https://github.com/sundy-li)). +* Added new in-memory format of parts in `MergeTree`-family tables, which stores data in memory. Parts are written on disk at first merge. Part will be created in in-memory format if its size in rows or bytes is below thresholds `min_rows_for_compact_part` and `min_bytes_for_compact_part`. Also optional support of Write-Ahead-Log is available, which is enabled by default and is controlled by setting `in_memory_parts_enable_wal`. [#10697](https://github.com/ClickHouse/ClickHouse/pull/10697) ([Anton Popov](https://github.com/CurtizJ)). +* Add -Distinct combinator for aggregate functions. [#10930](https://github.com/ClickHouse/ClickHouse/pull/10930) ([Sofia Antipushina](https://github.com/Sonichka1311)). +* Support table engine mongo(host:port, database, collection, user, password). [#10931](https://github.com/ClickHouse/ClickHouse/pull/10931) ([ageraab](https://github.com/ageraab)). +* Add storage RabbitMQ. [#11069](https://github.com/ClickHouse/ClickHouse/pull/11069) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Opt-in settings to send crash reports to the ClickHouse core team via [Sentry](https://sentry.io). [#11300](https://github.com/ClickHouse/ClickHouse/pull/11300) ([Ivan Blinkov](https://github.com/blinkov)). +* Add ORCBlockOutputFormat. [#11662](https://github.com/ClickHouse/ClickHouse/pull/11662) ([Kruglov Pavel](https://github.com/Avogar)). +* `max_thread_pool_size` config for changing the maximum number of Threads in Global Thread Pool. [#11668](https://github.com/ClickHouse/ClickHouse/pull/11668) ([Bharat Nallan](https://github.com/bharatnc)). +* Initial implementation of `EXPLAIN` query. Syntax: `EXPLAIN SELECT ...`. This fixes [#1118](https://github.com/ClickHouse/ClickHouse/issues/1118). [#11873](https://github.com/ClickHouse/ClickHouse/pull/11873) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Switched paths in S3 metadata to relative which allows to handle S3 blobs more easily. [#11892](https://github.com/ClickHouse/ClickHouse/pull/11892) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Avro UUID input/output support. [#11954](https://github.com/ClickHouse/ClickHouse/pull/11954) ([Andrew Onyshchuk](https://github.com/oandrew)). +* Added read-only MongoDB table engine. Allows reading flat (primitive types, not nested) fields. [#11983](https://github.com/ClickHouse/ClickHouse/pull/11983) ([alesapin](https://github.com/alesapin)). +* Add setting to fields not found in Avro schema. [#12007](https://github.com/ClickHouse/ClickHouse/pull/12007) ([Andrew Onyshchuk](https://github.com/oandrew)). +* add function `parseDateTimeBestEffortUS`. [#12028](https://github.com/ClickHouse/ClickHouse/pull/12028) ([flynn](https://github.com/ucasfl)). +* #4006 Support ALTER TABLE ... [ADD|MODIFY] COLUMN ... FIRST. [#12073](https://github.com/ClickHouse/ClickHouse/pull/12073) ([Winter Zhang](https://github.com/zhang2014)). +* Add a function initializedAggregation to initialize an aggregation based on a single value. [#12109](https://github.com/ClickHouse/ClickHouse/pull/12109) ([Guillaume Tassery](https://github.com/YiuRULE)). +* Support RIGHT and FULL JOIN with `set join_algorithm=partial_merge`. Only ALL strictness is supported (ANY, SEMI, ANTI, ASOF are not). [#12118](https://github.com/ClickHouse/ClickHouse/pull/12118) ([Artem Zuikov](https://github.com/4ertus2)). +* Implementation of PostgreSQL-like ILIKE operator for [#11710](https://github.com/ClickHouse/ClickHouse/issues/11710). [#12125](https://github.com/ClickHouse/ClickHouse/pull/12125) ([Mike Kot](https://github.com/myrrc)). + +#### Performance Improvement +* Allow to use direct_io and mmap_io for secondary indices if the settings `min_bytes_to_use_direct_io` or `min_bytes_to_use_mmap_io` are configured. [#11955](https://github.com/ClickHouse/ClickHouse/pull/11955) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix estimation of the number of marks while reading from MergeTree. This is needed to correctly handle the settings `merge_tree_max_rows_to_use_cache`, `merge_tree_max_bytes_to_use_cache`, `merge_tree_min_rows_for_concurrent_read`, `merge_tree_min_bytes_for_concurrent_read`, `merge_tree_min_rows_for_seek`, `merge_tree_min_bytes_for_seek`. Now settings `min_bytes_to_use_mmap_io` also applied to read index and compact parts in MergeTree table engines family. [#11970](https://github.com/ClickHouse/ClickHouse/pull/11970) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix "[#10574](https://github.com/ClickHouse/ClickHouse/issues/10574) Index not used for IN operator with literals", performance regression introduced around v19.3. [#12062](https://github.com/ClickHouse/ClickHouse/pull/12062) ([nvartolomei](https://github.com/nvartolomei)). +* Remove injective functions inside `uniq*()` if `set optimize_injective_functions_inside_uniq=1`. [#12337](https://github.com/ClickHouse/ClickHouse/pull/12337) ([Artem Zuikov](https://github.com/4ertus2)). + +#### Improvement +* Add number of errors to ignore while choosing replicas (`distributed_replica_error_ignore`). [#11669](https://github.com/ClickHouse/ClickHouse/pull/11669) ([Azat Khuzhin](https://github.com/azat)). +* Improved performace of 'ORDER BY' and 'GROUP BY' by prefix of sorting key. [#11696](https://github.com/ClickHouse/ClickHouse/pull/11696) ([Anton Popov](https://github.com/CurtizJ)). +* - Add `optimize_skip_unused_shards_nesting` (allows control nesting level for shards skipping optimization) - Add `force_skip_optimize_shards_nesting` (allows control nesting level for checking was shards skipped or not) - Deprecate `force_optimize_skip_unused_shards_no_nested` (`force_skip_optimize_shards_nesting` should be used instead) - Disable `optimize_skip_unused_shards` if sharding_key has non-deterministic func (i.e. `rand()`, note that this does not changes anything for INSERT side). [#11715](https://github.com/ClickHouse/ClickHouse/pull/11715) ([Azat Khuzhin](https://github.com/azat)). +* Multiversion metadata for storages without structure locks. [#11745](https://github.com/ClickHouse/ClickHouse/pull/11745) ([alesapin](https://github.com/alesapin)). +* Slightly relax the validation of ODBC connection string. If the hostname or username contains only word characters along with `.` and `-`, don't put it into curly braces. It is needed, because some ODBC drivers (e.g. PostgreSQL) don't understand when hostname is enclosed in curly braces. [#11845](https://github.com/ClickHouse/ClickHouse/pull/11845) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support parse UUID without separator(separators are always removed in most implementations, this is helpful for users to write data). [#11856](https://github.com/ClickHouse/ClickHouse/pull/11856) ([Winter Zhang](https://github.com/zhang2014)). +* Support `SIGNED` and `UNSIGNED` modifiers of standard integer types (`BIGINT`, `INT`, ...) for compatibility with MySQL. [#11858](https://github.com/ClickHouse/ClickHouse/pull/11858) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Allow to use `sumWithOverflow` as `SimpleAggregateFunction`. Closes [#8053](https://github.com/ClickHouse/ClickHouse/issues/8053). [#11865](https://github.com/ClickHouse/ClickHouse/pull/11865) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add FixedString support in Hashing functions. [#11878](https://github.com/ClickHouse/ClickHouse/pull/11878) ([flynn](https://github.com/ucasfl)). +* Rewrite code for `optimize_arithmetic_operations_in_aggregate_functions` optimisation. [#11899](https://github.com/ClickHouse/ClickHouse/pull/11899) ([Artem Zuikov](https://github.com/4ertus2)). +* Improve path concatenation and fix double slashed paths using std::filesystem::path instead of std::string in `DatabaseOrdinary.cpp`. [#11900](https://github.com/ClickHouse/ClickHouse/pull/11900) ([Bharat Nallan](https://github.com/bharatnc)). +* Deprecate the old regular style and use the new globalVariable method [#11832](https://github.com/ClickHouse/ClickHouse/issues/11832). [#11901](https://github.com/ClickHouse/ClickHouse/pull/11901) ([BohuTANG](https://github.com/BohuTANG)). +* related to [issue 9797](https://github.com/ClickHouse/ClickHouse/issues/9797). [#11923](https://github.com/ClickHouse/ClickHouse/pull/11923) ([flynn](https://github.com/ucasfl)). +* `system.tables` now considers column capacities for Memory and Buffer table engines, which is better approximation for resident memory size. [#11935](https://github.com/ClickHouse/ClickHouse/pull/11935) ([Max Akhmedov](https://github.com/zlobober)). +* Add CPU frequencies to system.asynchronous_metrics. Make the metric collection period configurable. [#11972](https://github.com/ClickHouse/ClickHouse/pull/11972) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow to perform "metadata-only" ALTER of partition key such as adding more elements to Enum data type. This fixes [#7513](https://github.com/ClickHouse/ClickHouse/issues/7513). [#11973](https://github.com/ClickHouse/ClickHouse/pull/11973) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add replica priority for load_balancing (for manual prioritization of the load balancing). [#11995](https://github.com/ClickHouse/ClickHouse/pull/11995) ([Azat Khuzhin](https://github.com/azat)). +* Support MySQL engine reading Enums type [#3985](https://github.com/ClickHouse/ClickHouse/issues/3985). [#11996](https://github.com/ClickHouse/ClickHouse/pull/11996) ([BohuTANG](https://github.com/BohuTANG)). +* Implemented single part uploads for DiskS3. [#12026](https://github.com/ClickHouse/ClickHouse/pull/12026) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Moved useless S3 logging to TRACE level. [#12067](https://github.com/ClickHouse/ClickHouse/pull/12067) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Improves `REVOKE` command: now it requires grant/admin option for only access which will be revoked. For example, to execute `REVOKE ALL ON *.* FROM user1` now it doesn't require to have full access rights granted with grant option. Added command `REVOKE ALL FROM user1` - it revokes all granted roles from `user1`. [#12083](https://github.com/ClickHouse/ClickHouse/pull/12083) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add 'type' column in system.disks. [#12115](https://github.com/ClickHouse/ClickHouse/pull/12115) ([ianton-ru](https://github.com/ianton-ru)). +* Added support for `%g` (two digit ISO year) and `%G` (four digit ISO year) substitutions in `formatDateTime` function. [#12136](https://github.com/ClickHouse/ClickHouse/pull/12136) ([vivarum](https://github.com/vivarum)). +* Add `KILL QUERY [connection_id]` for the MySQL client/driver to cancel the long query, issue [#12038](https://github.com/ClickHouse/ClickHouse/issues/12038). [#12152](https://github.com/ClickHouse/ClickHouse/pull/12152) ([BohuTANG](https://github.com/BohuTANG)). +* 1. Support MySQL 'SELECT DATABASE()' [#9336](https://github.com/ClickHouse/ClickHouse/issues/9336) 2. Add MySQL replacement query integration test. [#12314](https://github.com/ClickHouse/ClickHouse/pull/12314) ([BohuTANG](https://github.com/BohuTANG)). +* This setting allows to chose charset for printing grids (either utf8 or ascii). [#12372](https://github.com/ClickHouse/ClickHouse/pull/12372) ([Sabyanin Maxim](https://github.com/s-mx)). +* Write the detail exception message to the client instead of 'MySQL server has gone away'. [#12383](https://github.com/ClickHouse/ClickHouse/pull/12383) ([BohuTANG](https://github.com/BohuTANG)). +* lifetime_rows/lifetime_bytes for Buffer engine. [#12421](https://github.com/ClickHouse/ClickHouse/pull/12421) ([Azat Khuzhin](https://github.com/azat)). + +#### Bug Fix +* Fix unexpected behaviour of queries like `SELECT *, xyz.*` which were success while an error expected. [#11753](https://github.com/ClickHouse/ClickHouse/pull/11753) ([hexiaoting](https://github.com/hexiaoting)). +* Fix wrong result for `if()` with NULLs in condition. [#11807](https://github.com/ClickHouse/ClickHouse/pull/11807) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix memory accounting via HTTP interface (can be significant with `wait_end_of_query=1`). [#11840](https://github.com/ClickHouse/ClickHouse/pull/11840) ([Azat Khuzhin](https://github.com/azat)). +* Fix rare crash caused by using `Nullable` column in prewhere condition. Continuation of [#11608](https://github.com/ClickHouse/ClickHouse/issues/11608). [#11869](https://github.com/ClickHouse/ClickHouse/pull/11869) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix potential floating point exception when parsing DateTime64. This fixes [#11374](https://github.com/ClickHouse/ClickHouse/issues/11374). [#11875](https://github.com/ClickHouse/ClickHouse/pull/11875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed bug with no moves when changing storage policy from default one. [#11893](https://github.com/ClickHouse/ClickHouse/pull/11893) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix rare crash caused by using Nullable column in prewhere condition. Continuation of [#11869](https://github.com/ClickHouse/ClickHouse/issues/11869). [#11895](https://github.com/ClickHouse/ClickHouse/pull/11895) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Keep aliases for substitutions in query (parametrized queries). This fixes [#11914](https://github.com/ClickHouse/ClickHouse/issues/11914). [#11916](https://github.com/ClickHouse/ClickHouse/pull/11916) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix unitialized memory in partitions serialization. [#11919](https://github.com/ClickHouse/ClickHouse/pull/11919) ([alesapin](https://github.com/alesapin)). +* Use the correct current database for checking access rights after statement `USE database`. [#11920](https://github.com/ClickHouse/ClickHouse/pull/11920) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed `Query parameter was not set` in `Values` format. Fixes [#11918](https://github.com/ClickHouse/ClickHouse/issues/11918). [#11936](https://github.com/ClickHouse/ClickHouse/pull/11936) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix race condition in extractAllGroups* functions. [#11949](https://github.com/ClickHouse/ClickHouse/pull/11949) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make mmap IO work again (experimental). Continuation of [#8520](https://github.com/ClickHouse/ClickHouse/issues/8520). [#11953](https://github.com/ClickHouse/ClickHouse/pull/11953) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong setting name in log message at server startup. [#11997](https://github.com/ClickHouse/ClickHouse/pull/11997) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Partial revokes work correctly in complex cases as well, for example. [#12002](https://github.com/ClickHouse/ClickHouse/pull/12002) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix potential floating point exception. This closes [#11378](https://github.com/ClickHouse/ClickHouse/issues/11378). [#12005](https://github.com/ClickHouse/ClickHouse/pull/12005) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid returning wrong number of geohashes in function `geoHashesInBox` due to accumulation of floating point error. This fixes [#11369](https://github.com/ClickHouse/ClickHouse/issues/11369). [#12006](https://github.com/ClickHouse/ClickHouse/pull/12006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potential array size overflow in generateRandom that may lead to crash. This fixes [#11371](https://github.com/ClickHouse/ClickHouse/issues/11371). [#12013](https://github.com/ClickHouse/ClickHouse/pull/12013) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix calculation of access rights when allow_ddl=0. [#12015](https://github.com/ClickHouse/ClickHouse/pull/12015) ([Vitaly Baranov](https://github.com/vitlibar)). +* When adding floating point number of intervals to date/datetime, the result may be calculated incorrectly. This fixes [#11377](https://github.com/ClickHouse/ClickHouse/issues/11377). [#12018](https://github.com/ClickHouse/ClickHouse/pull/12018) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* A query with function `neighbor` as the only returned expression may return empty result if the function is called with offset `-9223372036854775808`. This fixes [#11367](https://github.com/ClickHouse/ClickHouse/issues/11367). [#12019](https://github.com/ClickHouse/ClickHouse/pull/12019) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not mention in changelog, because the bug did not come to release. Fix potential crash when doing ORDER BY multiple columns with specified COLLATE on one of the column when this column is constant. This fixes [#11379](https://github.com/ClickHouse/ClickHouse/issues/11379). The bug was introduced in [#11006](https://github.com/ClickHouse/ClickHouse/issues/11006) in version 20.5. [#12020](https://github.com/ClickHouse/ClickHouse/pull/12020) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong result and potential crash when invoking function `if` with arguments of type `FixedString` with different sizes. This fixes [#11362](https://github.com/ClickHouse/ClickHouse/issues/11362). [#12021](https://github.com/ClickHouse/ClickHouse/pull/12021) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix calculation of access rights when allow_introspection_functions=0. [#12031](https://github.com/ClickHouse/ClickHouse/pull/12031) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix crash in JOIN with LowCardinality type with `join_algorithm=partial_merge`. [#12035](https://github.com/ClickHouse/ClickHouse/pull/12035) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix incorrect comparison of tuples with `Nullable` columns. Fixes [#11985](https://github.com/ClickHouse/ClickHouse/issues/11985). [#12039](https://github.com/ClickHouse/ClickHouse/pull/12039) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix constraints check if constraint is a constant expression. This fixes [#11360](https://github.com/ClickHouse/ClickHouse/issues/11360). [#12042](https://github.com/ClickHouse/ClickHouse/pull/12042) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `topK` aggregate function return Enum for Enum types. This fixes [#3740](https://github.com/ClickHouse/ClickHouse/issues/3740). [#12043](https://github.com/ClickHouse/ClickHouse/pull/12043) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Parse tables metadata in parallel when loading database. This fixes slow server startup when there are large number of tables. [#12045](https://github.com/ClickHouse/ClickHouse/pull/12045) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error `Cannot capture column` for higher-order functions with `Tuple(LowCardinality)` argument. Fixes [#9766](https://github.com/ClickHouse/ClickHouse/issues/9766). [#12055](https://github.com/ClickHouse/ClickHouse/pull/12055) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix error `Expected single dictionary argument for function` for function `defaultValueOfArgumentType` with `LowCardinality` type. Fixes [#11808](https://github.com/ClickHouse/ClickHouse/issues/11808). [#12056](https://github.com/ClickHouse/ClickHouse/pull/12056) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible crash while using wrong type for `PREWHERE`. Fixes [#12053](https://github.com/ClickHouse/ClickHouse/issues/12053), [#12060](https://github.com/ClickHouse/ClickHouse/issues/12060). [#12060](https://github.com/ClickHouse/ClickHouse/pull/12060) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix SIGSEGV in StorageKafka on DROP TABLE. [#12075](https://github.com/ClickHouse/ClickHouse/pull/12075) ([Azat Khuzhin](https://github.com/azat)). +* Fix unnecessary limiting the number of threads for selects from `VIEW`. Fixes [#11937](https://github.com/ClickHouse/ClickHouse/issues/11937). [#12085](https://github.com/ClickHouse/ClickHouse/pull/12085) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix empty `result_rows` and `result_bytes` metrics in `system.quey_log` for selects. Fixes [#11595](https://github.com/ClickHouse/ClickHouse/issues/11595). [#12089](https://github.com/ClickHouse/ClickHouse/pull/12089) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix segfault with `-StateResample` combinators. [#12092](https://github.com/ClickHouse/ClickHouse/pull/12092) ([Anton Popov](https://github.com/CurtizJ)). +* Fix performance for selects with `UNION` caused by wrong limit for the total number of threads. Fixes [#12030](https://github.com/ClickHouse/ClickHouse/issues/12030). [#12103](https://github.com/ClickHouse/ClickHouse/pull/12103) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Format `Parquet` now properly works with `LowCardinality` and `LowCardinality(Nullable)` types. Fixes [#12086](https://github.com/ClickHouse/ClickHouse/issues/12086), [#8406](https://github.com/ClickHouse/ClickHouse/issues/8406). [#12108](https://github.com/ClickHouse/ClickHouse/pull/12108) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix handling dependency of table with ENGINE=Dictionary on dictionary. This fixes [#10994](https://github.com/ClickHouse/ClickHouse/issues/10994). This fixes [#10397](https://github.com/ClickHouse/ClickHouse/issues/10397). [#12116](https://github.com/ClickHouse/ClickHouse/pull/12116) ([Vitaly Baranov](https://github.com/vitlibar)). +* Avoid "There is no query" exception for materialized views with joins or with subqueries attached to system logs (system.query_log, metric_log, etc) or to engine=Buffer underlying table. [#12120](https://github.com/ClickHouse/ClickHouse/pull/12120) ([filimonov](https://github.com/filimonov)). +* Fix bug which leads to incorrect table metadata in ZooKeepeer for ReplicatedVersionedCollapsingMergeTree tables. Fixes [#12093](https://github.com/ClickHouse/ClickHouse/issues/12093). [#12121](https://github.com/ClickHouse/ClickHouse/pull/12121) ([alesapin](https://github.com/alesapin)). +* Normalize "pid" file handling. In previous versions the server may refuse to start if it was killed without proper shutdown and if there is another process that has the same pid as previously runned server. Also pid file may be removed in unsuccessful server startup even if there is another server running. This fixes [#3501](https://github.com/ClickHouse/ClickHouse/issues/3501). [#12133](https://github.com/ClickHouse/ClickHouse/pull/12133) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potential infinite loop in `greatCircleDistance`, `geoDistance`. This fixes [#12117](https://github.com/ClickHouse/ClickHouse/issues/12117). [#12137](https://github.com/ClickHouse/ClickHouse/pull/12137) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potential overflow in integer division. This fixes [#12119](https://github.com/ClickHouse/ClickHouse/issues/12119). [#12140](https://github.com/ClickHouse/ClickHouse/pull/12140) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad code in redundant ORDER BY optimization. The bug was introduced in [#10067](https://github.com/ClickHouse/ClickHouse/issues/10067). [#12148](https://github.com/ClickHouse/ClickHouse/pull/12148) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix transform of query to send to external DBMS (e.g. MySQL, ODBC) in presense of aliases. This fixes [#12032](https://github.com/ClickHouse/ClickHouse/issues/12032). [#12151](https://github.com/ClickHouse/ClickHouse/pull/12151) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong logic in ALTER DELETE that leads to deleting of records when condition evaluates to NULL. This fixes [#9088](https://github.com/ClickHouse/ClickHouse/issues/9088). This closes [#12106](https://github.com/ClickHouse/ClickHouse/issues/12106). [#12153](https://github.com/ClickHouse/ClickHouse/pull/12153) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't split the dictionary source's table name into schema and table name itself if ODBC connection doesn't support schema. [#12165](https://github.com/ClickHouse/ClickHouse/pull/12165) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed the behaviour when `SummingMergeTree` engine sums up columns from partition key. Added an exception in case of explicit definition of columns to sum which intersects with partition key columns. This fixes [#7867](https://github.com/ClickHouse/ClickHouse/issues/7867). [#12173](https://github.com/ClickHouse/ClickHouse/pull/12173) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix dictGet arguments check during GROUP BY injective functions elimination. [#12179](https://github.com/ClickHouse/ClickHouse/pull/12179) ([Azat Khuzhin](https://github.com/azat)). +* Cap max_memory_usage* limits to the process resident memory. [#12182](https://github.com/ClickHouse/ClickHouse/pull/12182) ([Azat Khuzhin](https://github.com/azat)). +* Fixed logical functions for UInt8 values when they are not equal to 0 or 1. [#12196](https://github.com/ClickHouse/ClickHouse/pull/12196) ([Alexander Kazakov](https://github.com/Akazz)). +* Fixed the behaviour when during multiple sequential inserts in `StorageFile` header for some special types was written more than once. This fixed [#6155](https://github.com/ClickHouse/ClickHouse/issues/6155). [#12197](https://github.com/ClickHouse/ClickHouse/pull/12197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed behaviour on reaching redirect limit in request to S3 storage. [#12256](https://github.com/ClickHouse/ClickHouse/pull/12256) ([ianton-ru](https://github.com/ianton-ru)). +* Not for changelog. Cherry-pick after [#12196](https://github.com/ClickHouse/ClickHouse/issues/12196). [#12271](https://github.com/ClickHouse/ClickHouse/pull/12271) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Implement conversions to the common type for LowCardinality types. This allows to execute UNION ALL of tables with columns of LowCardinality and other columns. This fixes [#8212](https://github.com/ClickHouse/ClickHouse/issues/8212). This fixes [#4342](https://github.com/ClickHouse/ClickHouse/issues/4342). [#12275](https://github.com/ClickHouse/ClickHouse/pull/12275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The function `arrayFill` worked incorrectly for empty arrays that may lead to crash. This fixes [#12263](https://github.com/ClickHouse/ClickHouse/issues/12263). [#12279](https://github.com/ClickHouse/ClickHouse/pull/12279) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Show error after TrieDictionary failed to load. [#12290](https://github.com/ClickHouse/ClickHouse/pull/12290) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix typo in setting name. [#12292](https://github.com/ClickHouse/ClickHouse/pull/12292) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Some threads might randomly hang for a few seconds during DNS cache updating. It's fixed. [#12296](https://github.com/ClickHouse/ClickHouse/pull/12296) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#12724](https://github.com/ClickHouse/ClickHouse/issues/12724): kafka: fix SIGSEGV if there is an message with error in the middle of the batch. [#12302](https://github.com/ClickHouse/ClickHouse/pull/12302) ([Azat Khuzhin](https://github.com/azat)). +* Fix TTL after renaming column, on which depends TTL expression. [#12304](https://github.com/ClickHouse/ClickHouse/pull/12304) ([Anton Popov](https://github.com/CurtizJ)). +* Avoid "bad cast" exception when there is an expression that filters data by virtual columns (like `_table` in `Merge` tables) or by "index" columns in system tables such as filtering by database name when querying from `system.tables`, and this expression returns `Nullable` type. This fixes [#12166](https://github.com/ClickHouse/ClickHouse/issues/12166). [#12305](https://github.com/ClickHouse/ClickHouse/pull/12305) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix order of columns in `WITH FILL` modifier. Previously order of columns of `ORDER BY` statement wasn't respected. [#12306](https://github.com/ClickHouse/ClickHouse/pull/12306) ([Anton Popov](https://github.com/CurtizJ)). +* When using codec `Delta` or `DoubleDelta` with non fixed width types, exception with code `LOGICAL_ERROR` was returned instead of exception with code `BAD_ARGUMENTS` (we ensure that exceptions with code logical error never happen). This fixes [#12110](https://github.com/ClickHouse/ClickHouse/issues/12110). [#12308](https://github.com/ClickHouse/ClickHouse/pull/12308) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix very rare race condition in ReplicatedMergeTreeQueue. [#12315](https://github.com/ClickHouse/ClickHouse/pull/12315) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error message and exit codes for `ALTER RENAME COLUMN` queries, when `RENAME` is not allowed. Fixes [#12301](https://github.com/ClickHouse/ClickHouse/issues/12301) and [#12303](https://github.com/ClickHouse/ClickHouse/issues/12303). [#12335](https://github.com/ClickHouse/ClickHouse/pull/12335) ([alesapin](https://github.com/alesapin)). +* Fix TOTALS/ROLLUP/CUBE for aggregate functions with `-State` and `Nullable` arguments. This fixes [#12163](https://github.com/ClickHouse/ClickHouse/issues/12163). [#12376](https://github.com/ClickHouse/ClickHouse/pull/12376) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to CLEAR column even if there are depending DEFAULT expressions. This fixes [#12333](https://github.com/ClickHouse/ClickHouse/issues/12333). [#12378](https://github.com/ClickHouse/ClickHouse/pull/12378) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12723](https://github.com/ClickHouse/ClickHouse/issues/12723): If MergeTree table does not contain ORDER BY or PARTITION BY, it was possible to request ALTER to CLEAR all the columns and ALTER will stuck. Fixed [#7941](https://github.com/ClickHouse/ClickHouse/issues/7941). [#12382](https://github.com/ClickHouse/ClickHouse/pull/12382) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid exception when negative or floating point constant is used in WHERE condition for indexed tables. This fixes [#11905](https://github.com/ClickHouse/ClickHouse/issues/11905). [#12384](https://github.com/ClickHouse/ClickHouse/pull/12384) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Additional check for arguments of bloom filter index. This fixes [#11408](https://github.com/ClickHouse/ClickHouse/issues/11408). [#12388](https://github.com/ClickHouse/ClickHouse/pull/12388) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Reverts change introduced in [#11079](https://github.com/ClickHouse/ClickHouse/issues/11079) to resolve [#12098](https://github.com/ClickHouse/ClickHouse/issues/12098). [#12397](https://github.com/ClickHouse/ClickHouse/pull/12397) ([Mike Kot](https://github.com/myrrc)). +* Fixed possible segfault if StorageMerge. Closes [#12054](https://github.com/ClickHouse/ClickHouse/issues/12054). [#12401](https://github.com/ClickHouse/ClickHouse/pull/12401) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#12725](https://github.com/ClickHouse/ClickHouse/issues/12725): Fix crash in JOIN with dictionary when we are joining over expression of dictionary key: `t JOIN dict ON expr(dict.id) = t.id`. Disable dictionary join optimisation for this case. [#12458](https://github.com/ClickHouse/ClickHouse/pull/12458) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#12803](https://github.com/ClickHouse/ClickHouse/issues/12803): Fix SETTINGS parse after FORMAT. [#12480](https://github.com/ClickHouse/ClickHouse/pull/12480) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#12862](https://github.com/ClickHouse/ClickHouse/issues/12862): Fixed performance issue, while reading from compact parts. [#12492](https://github.com/ClickHouse/ClickHouse/pull/12492) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#12722](https://github.com/ClickHouse/ClickHouse/issues/12722): Fixing race condition in live view tables which could cause data duplication. [#12519](https://github.com/ClickHouse/ClickHouse/pull/12519) ([vzakaznikov](https://github.com/vzakaznikov)). +* Backported in [#12721](https://github.com/ClickHouse/ClickHouse/issues/12721): Now ClickHouse will recalculate checksums for parts when file `checksums.txt` is absent. Broken since [#9827](https://github.com/ClickHouse/ClickHouse/issues/9827). [#12545](https://github.com/ClickHouse/ClickHouse/pull/12545) ([alesapin](https://github.com/alesapin)). +* Backported in [#12695](https://github.com/ClickHouse/ClickHouse/issues/12695): Fix error `Output of TreeExecutor is not sorted` for `OPTIMIZE DEDUPLICATE`. Fixes [#11572](https://github.com/ClickHouse/ClickHouse/issues/11572). [#12613](https://github.com/ClickHouse/ClickHouse/pull/12613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#12699](https://github.com/ClickHouse/ClickHouse/issues/12699): Fix possible `Pipeline stuck` error for queries with external sorting. Fixes [#12617](https://github.com/ClickHouse/ClickHouse/issues/12617). [#12618](https://github.com/ClickHouse/ClickHouse/pull/12618) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#12720](https://github.com/ClickHouse/ClickHouse/issues/12720): Fix error message about adaptive granularity. [#12624](https://github.com/ClickHouse/ClickHouse/pull/12624) ([alesapin](https://github.com/alesapin)). +* Backported in [#12696](https://github.com/ClickHouse/ClickHouse/issues/12696): Better exception message in disk access storage. [#12625](https://github.com/ClickHouse/ClickHouse/pull/12625) ([alesapin](https://github.com/alesapin)). +* Backported in [#12698](https://github.com/ClickHouse/ClickHouse/issues/12698): Exception `There is no supertype...` can be thrown during `ALTER ... UPDATE` in unexpected cases (e.g. when subtracting from UInt64 column). This fixes [#7306](https://github.com/ClickHouse/ClickHouse/issues/7306). This fixes [#4165](https://github.com/ClickHouse/ClickHouse/issues/4165). [#12633](https://github.com/ClickHouse/ClickHouse/pull/12633) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12697](https://github.com/ClickHouse/ClickHouse/issues/12697): Add support for function `if` with `Array(UUID)` arguments. This fixes [#11066](https://github.com/ClickHouse/ClickHouse/issues/11066). [#12648](https://github.com/ClickHouse/ClickHouse/pull/12648) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#12971](https://github.com/ClickHouse/ClickHouse/issues/12971): Fix SIGSEGV in StorageKafka when broker is unavailable (and not only). [#12658](https://github.com/ClickHouse/ClickHouse/pull/12658) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#12858](https://github.com/ClickHouse/ClickHouse/issues/12858): fixes [#10572](https://github.com/ClickHouse/ClickHouse/issues/10572) fix bloom filter index with const expression. [#12659](https://github.com/ClickHouse/ClickHouse/pull/12659) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#12868](https://github.com/ClickHouse/ClickHouse/issues/12868): fixes [#12293](https://github.com/ClickHouse/ClickHouse/issues/12293) allow push predicate when subquery contains with clause. [#12663](https://github.com/ClickHouse/ClickHouse/pull/12663) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#12994](https://github.com/ClickHouse/ClickHouse/issues/12994): Fix optimization `optimize_move_functions_out_of_any=1` in case of `any(func())`. [#12664](https://github.com/ClickHouse/ClickHouse/pull/12664) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#12864](https://github.com/ClickHouse/ClickHouse/issues/12864): Fix memory tracking for input_format_parallel_parsing (by attaching thread to group). [#12672](https://github.com/ClickHouse/ClickHouse/pull/12672) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13187](https://github.com/ClickHouse/ClickHouse/issues/13187): Fix performance with large tuples, which are interpreted as functions in `IN` section. The case when user write `WHERE x IN tuple(1, 2, ...)` instead of `WHERE x IN (1, 2, ...)` for some obscure reason. [#12700](https://github.com/ClickHouse/ClickHouse/pull/12700) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#13095](https://github.com/ClickHouse/ClickHouse/issues/13095): Fix CAST(Nullable(String), Enum()). [#12745](https://github.com/ClickHouse/ClickHouse/pull/12745) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13011](https://github.com/ClickHouse/ClickHouse/issues/13011): Fix rare bug when `ALTER DELETE` and `ALTER MODIFY COLUMN` queries executed simultaneously as a single mutation. Bug leads to an incorrect amount of rows in `count.txt` and as a consequence incorrect data in part. Also, fix a small bug with simultaneous `ALTER RENAME COLUMN` and `ALTER ADD COLUMN`. [#12760](https://github.com/ClickHouse/ClickHouse/pull/12760) ([alesapin](https://github.com/alesapin)). +* Backported in [#13031](https://github.com/ClickHouse/ClickHouse/issues/13031): Corrected merge_with_ttl_timeout logic which did not work well when expiration affected more than one partition over one time interval. (Authored by @excitoon). [#12982](https://github.com/ClickHouse/ClickHouse/pull/12982) ([Alexander Kazakov](https://github.com/Akazz)). +* Backported in [#13049](https://github.com/ClickHouse/ClickHouse/issues/13049): Fix `Block structure mismatch` error for queries with `UNION` and `JOIN`. Fixes [#12602](https://github.com/ClickHouse/ClickHouse/issues/12602). [#12989](https://github.com/ClickHouse/ClickHouse/pull/12989) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13051](https://github.com/ClickHouse/ClickHouse/issues/13051): Fix crash which was possible for queries with `ORDER BY` tuple and small `LIMIT`. Fixes [#12623](https://github.com/ClickHouse/ClickHouse/issues/12623). [#13009](https://github.com/ClickHouse/ClickHouse/pull/13009) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13080](https://github.com/ClickHouse/ClickHouse/issues/13080): Add logging when the mutation is not running because of limited disk space or free threads in the background pool. [#13068](https://github.com/ClickHouse/ClickHouse/pull/13068) ([alesapin](https://github.com/alesapin)). +* Backported in [#13184](https://github.com/ClickHouse/ClickHouse/issues/13184): Fix error `Cannot convert column because it is constant but values of constants are different in source and result` for remote queries which use deterministic functions in scope of query, but not deterministic between queries, like `now()`, `now64()`, `randConstant()`. Fixes [#11327](https://github.com/ClickHouse/ClickHouse/issues/11327). [#13075](https://github.com/ClickHouse/ClickHouse/pull/13075) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13146](https://github.com/ClickHouse/ClickHouse/issues/13146): Fix wrong index analysis with functions. It could lead to pruning wrong parts, while reading from `MergeTree` tables. Fixes [#13060](https://github.com/ClickHouse/ClickHouse/issues/13060). Fixes [#12406](https://github.com/ClickHouse/ClickHouse/issues/12406). [#13081](https://github.com/ClickHouse/ClickHouse/pull/13081) ([Anton Popov](https://github.com/CurtizJ)). + +#### Build/Testing/Packaging Improvement +* Add simple GitHub hook script for the serverless environment. [#11605](https://github.com/ClickHouse/ClickHouse/pull/11605) ([alesapin](https://github.com/alesapin)). +* Send logs to client on fatal errors if possible. This will make test results more readable. [#11826](https://github.com/ClickHouse/ClickHouse/pull/11826) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow ClickHouse to run on Android. [#11894](https://github.com/ClickHouse/ClickHouse/pull/11894) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Clean up unused header files from `Obfuscator.cpp` and `DatabaseAtomic.cpp`. [#11922](https://github.com/ClickHouse/ClickHouse/pull/11922) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix some typos in code. [#12003](https://github.com/ClickHouse/ClickHouse/pull/12003) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Scripts for automated scheduled backporting based on PR labels. [#12029](https://github.com/ClickHouse/ClickHouse/pull/12029) ([Ivan](https://github.com/abyss7)). +* Add new type of tests based on Testflows framework. [#12090](https://github.com/ClickHouse/ClickHouse/pull/12090) ([vzakaznikov](https://github.com/vzakaznikov)). +* Install `ca-certificates` before the first `apt-get update` in Dockerfile. [#12095](https://github.com/ClickHouse/ClickHouse/pull/12095) ([Ivan Blinkov](https://github.com/blinkov)). +* Daily check by [GitHub CodeQL](https://securitylab.github.com/tools/codeql) security analysis tool that looks for [CWE](https://cwe.mitre.org/). [#12101](https://github.com/ClickHouse/ClickHouse/pull/12101) ([Ivan Blinkov](https://github.com/blinkov)). +* Regular check by [Anchore Container Analysis](https://docs.anchore.com) security analysis tool that looks for [CVE](https://cve.mitre.org/) in `clickhouse-server` Docker image. Also confirms that `Dockerfile` is buildable. Runs daily on `master` and on pull-requests to `Dockerfile`. [#12102](https://github.com/ClickHouse/ClickHouse/pull/12102) ([Ivan Blinkov](https://github.com/blinkov)). +* Add `UNBUNDLED` flag to `system.build_options` table. Move skip lists for `clickhouse-test` to clickhouse repo. [#12107](https://github.com/ClickHouse/ClickHouse/pull/12107) ([alesapin](https://github.com/alesapin)). +* Implement AST-based query fuzzing mode for clickhouse-client. See [this label](https://github.com/ClickHouse/ClickHouse/issues?q=label%3Afuzz+is%3Aissue) for the list of issues we recently found by fuzzing. Most of them were found by this tool, and a couple by SQLancer and `00746_sql_fuzzy.pl`. [#12111](https://github.com/ClickHouse/ClickHouse/pull/12111) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Perform an upgrade of system packages in the `clickhouse-server` docker image. [#12124](https://github.com/ClickHouse/ClickHouse/pull/12124) ([Ivan Blinkov](https://github.com/blinkov)). +* Added a showcase of the minimal Docker image without using any Linux distribution. [#12126](https://github.com/ClickHouse/ClickHouse/pull/12126) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor CMake fixes for UNBUNDLED build. [#12131](https://github.com/ClickHouse/ClickHouse/pull/12131) ([Matwey V. Kornilov](https://github.com/matwey)). +* Missed `` is required for `std::atomic<>`. [#12134](https://github.com/ClickHouse/ClickHouse/pull/12134) ([Matwey V. Kornilov](https://github.com/matwey)). +* Fix warnings from CodeQL. `CodeQL` is another static analyzer that we will use along with `clang-tidy` and `PVS-Studio` that we use already. [#12138](https://github.com/ClickHouse/ClickHouse/pull/12138) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Record additional detail on Dockerfile scan reports. [#12159](https://github.com/ClickHouse/ClickHouse/pull/12159) ([Ivan Blinkov](https://github.com/blinkov)). +* Place common docker compose files to integration docker container. [#12168](https://github.com/ClickHouse/ClickHouse/pull/12168) ([Ilya Yatsishin](https://github.com/qoega)). +* Remove verbosity from the binary builds. [#12174](https://github.com/ClickHouse/ClickHouse/pull/12174) ([alesapin](https://github.com/alesapin)). +* Remove strange file creation during build in `orc`. [#12258](https://github.com/ClickHouse/ClickHouse/pull/12258) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Now functional and stress tests will be able to run with old version of `clickhouse-test` script. [#12287](https://github.com/ClickHouse/ClickHouse/pull/12287) ([alesapin](https://github.com/alesapin)). +* Log sanitizer trap messages from separate thread. This will prevent possible deadlock under thread sanitizer. [#12313](https://github.com/ClickHouse/ClickHouse/pull/12313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added S3 HTTPS integration test. [#12412](https://github.com/ClickHouse/ClickHouse/pull/12412) ([Pavel Kovalenko](https://github.com/Jokser)). + +#### Other +* Update word break characters to match readline default - all non-alphanumeric characters. ... [#11975](https://github.com/ClickHouse/ClickHouse/pull/11975) ([Andrew Onyshchuk](https://github.com/oandrew)). + +#### NO CL CATEGORY + +* * Not for changelog. [#12265](https://github.com/ClickHouse/ClickHouse/pull/12265) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ... [#12431](https://github.com/ClickHouse/ClickHouse/pull/12431) ([Tom Bombadil](https://github.com/ithangzhou)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Bump certifi from 2020.4.5.2 to 2020.6.20 in /docs/tools/translate'. [#11853](https://github.com/ClickHouse/ClickHouse/pull/11853) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Merging [#10679](https://github.com/ClickHouse/ClickHouse/issues/10679)'. [#11896](https://github.com/ClickHouse/ClickHouse/pull/11896) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "[experiment] maybe fix warnings in integration tests"'. [#12011](https://github.com/ClickHouse/ClickHouse/pull/12011) ([alesapin](https://github.com/alesapin)). +* NO CL ENTRY: 'Bump idna from 2.9 to 2.10 in /docs/tools'. [#12024](https://github.com/ClickHouse/ClickHouse/pull/12024) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump python-slugify from 1.2.6 to 4.0.1 in /docs/tools'. [#12049](https://github.com/ClickHouse/ClickHouse/pull/12049) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). + diff --git a/docs/changelogs/v20.6.3.28-stable.md b/docs/changelogs/v20.6.3.28-stable.md new file mode 100644 index 00000000000..c8ca7db9bd8 --- /dev/null +++ b/docs/changelogs/v20.6.3.28-stable.md @@ -0,0 +1,17 @@ +### ClickHouse release v20.6.3.28-stable FIXME as compared to v20.6.2.15-prestable + +#### Bug Fix +* Backported in [#13301](https://github.com/ClickHouse/ClickHouse/issues/13301): The function `groupArrayMoving*` was not working for distributed queries. It's result was calculated within incorrect data type (without promotion to the largest type). The function `groupArrayMovingAvg` was returning integer number that was inconsistent with the `avg` function. This fixes [#12568](https://github.com/ClickHouse/ClickHouse/issues/12568). [#12622](https://github.com/ClickHouse/ClickHouse/pull/12622) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13029](https://github.com/ClickHouse/ClickHouse/issues/13029): CREATE USER IF NOT EXISTS now doesn't throw exception if the user exists. This fixes [#12507](https://github.com/ClickHouse/ClickHouse/issues/12507). [#12646](https://github.com/ClickHouse/ClickHouse/pull/12646) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#12983](https://github.com/ClickHouse/ClickHouse/issues/12983): Fix columns duplication for range hashed dictionary created from DDL query. This fixes [#10605](https://github.com/ClickHouse/ClickHouse/issues/10605). [#12857](https://github.com/ClickHouse/ClickHouse/pull/12857) ([alesapin](https://github.com/alesapin)). +* Backported in [#13222](https://github.com/ClickHouse/ClickHouse/issues/13222): Fix DateTime64 conversion functions with constant argument. [#13205](https://github.com/ClickHouse/ClickHouse/pull/13205) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13245](https://github.com/ClickHouse/ClickHouse/issues/13245): Fix assert in `arrayElement` function in case of array elements are Nullable and array subscript is also Nullable. This fixes [#12172](https://github.com/ClickHouse/ClickHouse/issues/12172). [#13224](https://github.com/ClickHouse/ClickHouse/pull/13224) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13267](https://github.com/ClickHouse/ClickHouse/issues/13267): Fix function if with nullable constexpr as cond that is not literal NULL. Fixes [#12463](https://github.com/ClickHouse/ClickHouse/issues/12463). [#13226](https://github.com/ClickHouse/ClickHouse/pull/13226) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13298](https://github.com/ClickHouse/ClickHouse/issues/13298): Fix potentially low performance and slightly incorrect result for `uniqExact`, `topK`, `sumDistinct` and similar aggregate functions called on Float types with NaN values. It also triggered assert in debug build. This fixes [#12491](https://github.com/ClickHouse/ClickHouse/issues/12491). [#13254](https://github.com/ClickHouse/ClickHouse/pull/13254) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13316](https://github.com/ClickHouse/ClickHouse/issues/13316): The server may crash if user passed specifically crafted arguments to the function `h3ToChildren`. This fixes [#13275](https://github.com/ClickHouse/ClickHouse/issues/13275). [#13277](https://github.com/ClickHouse/ClickHouse/pull/13277) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13329](https://github.com/ClickHouse/ClickHouse/issues/13329): Fix possible error `Totals having transform was already added to pipeline` in case of a query from delayed replica. [#13290](https://github.com/ClickHouse/ClickHouse/pull/13290) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13317](https://github.com/ClickHouse/ClickHouse/issues/13317): Fix crash in `LEFT ASOF JOIN` with `join_use_nulls=1`. [#13291](https://github.com/ClickHouse/ClickHouse/pull/13291) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#13375](https://github.com/ClickHouse/ClickHouse/issues/13375): Fix segfault when function `groupArrayMovingSum` deserializes empty state. Fixes [#13339](https://github.com/ClickHouse/ClickHouse/issues/13339). [#13341](https://github.com/ClickHouse/ClickHouse/pull/13341) ([alesapin](https://github.com/alesapin)). +* Backported in [#13432](https://github.com/ClickHouse/ClickHouse/issues/13432): Fix PrettyCompactMonoBlock for clickhouse-local. Fix extremes/totals with PrettyCompactMonoBlock. Fixes [#7746](https://github.com/ClickHouse/ClickHouse/issues/7746). [#13394](https://github.com/ClickHouse/ClickHouse/pull/13394) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13443](https://github.com/ClickHouse/ClickHouse/issues/13443): Fix `aggregate function any(x) is found inside another aggregate function in query` error with `SET optimize_move_functions_out_of_any = 1` and aliases inside `any()`. [#13419](https://github.com/ClickHouse/ClickHouse/pull/13419) ([Artem Zuikov](https://github.com/4ertus2)). + diff --git a/docs/changelogs/v20.6.4.44-stable.md b/docs/changelogs/v20.6.4.44-stable.md new file mode 100644 index 00000000000..828f95f5144 --- /dev/null +++ b/docs/changelogs/v20.6.4.44-stable.md @@ -0,0 +1,22 @@ +### ClickHouse release v20.6.4.44-stable FIXME as compared to v20.6.3.28-stable + +#### Improvement +* Backported in [#13919](https://github.com/ClickHouse/ClickHouse/issues/13919): Fix data race in `lgamma` function. This race was caught only in `tsan`, no side effects a really happened. [#13842](https://github.com/ClickHouse/ClickHouse/pull/13842) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Bug Fix +* Backported in [#13560](https://github.com/ClickHouse/ClickHouse/issues/13560): Fix access to redis dictionary after connection was dropped once. It may happen with `cache` and `direct` dictionary layouts. [#13082](https://github.com/ClickHouse/ClickHouse/pull/13082) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#13507](https://github.com/ClickHouse/ClickHouse/issues/13507): Fix parsing row policies from users.xml when names of databases or tables contain dots. This fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779), [#12527](https://github.com/ClickHouse/ClickHouse/issues/12527). [#13199](https://github.com/ClickHouse/ClickHouse/pull/13199) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#13357](https://github.com/ClickHouse/ClickHouse/issues/13357): AvroConfluent: Skip Kafka tombstone records AvroConfluent: Support skipping broken records ... [#13203](https://github.com/ClickHouse/ClickHouse/pull/13203) ([Andrew Onyshchuk](https://github.com/oandrew)). +* Backported in [#13350](https://github.com/ClickHouse/ClickHouse/issues/13350): Return passed number for numbers with MSB set in roundUpToPowerOfTwoOrZero(). [#13234](https://github.com/ClickHouse/ClickHouse/pull/13234) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13479](https://github.com/ClickHouse/ClickHouse/issues/13479): Fix queries with constant columns and `ORDER BY` prefix of primary key. [#13396](https://github.com/ClickHouse/ClickHouse/pull/13396) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#13487](https://github.com/ClickHouse/ClickHouse/issues/13487): Fix empty output for `Arrow` and `Parquet` formats in case if query return zero rows. It was done because empty output is not valid for this formats. [#13399](https://github.com/ClickHouse/ClickHouse/pull/13399) ([hcz](https://github.com/hczhcz)). +* Backported in [#13524](https://github.com/ClickHouse/ClickHouse/issues/13524): Fix possible race in `StorageMemory`. https://clickhouse-test-reports.s3.yandex.net/0/9cac8a7244063d2092ad25d45502611e18d3749c/stress_test_(thread)/stderr.log Have no idea how to write a test. [#13416](https://github.com/ClickHouse/ClickHouse/pull/13416) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13483](https://github.com/ClickHouse/ClickHouse/issues/13483): Fix invalid return type for comparison of tuples with `NULL` elements. Fixes [#12461](https://github.com/ClickHouse/ClickHouse/issues/12461). [#13420](https://github.com/ClickHouse/ClickHouse/pull/13420) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13462](https://github.com/ClickHouse/ClickHouse/issues/13462): Fix error in `parseDateTimeBestEffort` function when unix timestamp was passed as an argument. This fixes [#13362](https://github.com/ClickHouse/ClickHouse/issues/13362). [#13441](https://github.com/ClickHouse/ClickHouse/pull/13441) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13552](https://github.com/ClickHouse/ClickHouse/issues/13552): Fix secondary indices corruption in compact parts. [#13538](https://github.com/ClickHouse/ClickHouse/pull/13538) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#13662](https://github.com/ClickHouse/ClickHouse/issues/13662): Concurrent `ALTER ... REPLACE/MOVE PARTITION ...` queries might cause deadlock. It's fixed. [#13626](https://github.com/ClickHouse/ClickHouse/pull/13626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#13720](https://github.com/ClickHouse/ClickHouse/issues/13720): Fix crash in JOIN with StorageMerge and `set enable_optimize_predicate_expression=1`. [#13679](https://github.com/ClickHouse/ClickHouse/pull/13679) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#13703](https://github.com/ClickHouse/ClickHouse/issues/13703): Do not optimize any(arrayJoin()) -> arrayJoin() under optimize_move_functions_out_of_any. [#13681](https://github.com/ClickHouse/ClickHouse/pull/13681) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13733](https://github.com/ClickHouse/ClickHouse/issues/13733): Fix incorrect message in `clickhouse-server.init` while checking user and group. [#13711](https://github.com/ClickHouse/ClickHouse/pull/13711) ([ylchou](https://github.com/ylchou)). +* Backported in [#13902](https://github.com/ClickHouse/ClickHouse/issues/13902): Fix incorrect sorting for `FixedString` columns. Fixes [#13182](https://github.com/ClickHouse/ClickHouse/issues/13182). [#13887](https://github.com/ClickHouse/ClickHouse/pull/13887) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v20.6.5.8-stable.md b/docs/changelogs/v20.6.5.8-stable.md new file mode 100644 index 00000000000..16462de13c2 --- /dev/null +++ b/docs/changelogs/v20.6.5.8-stable.md @@ -0,0 +1,33 @@ +### ClickHouse release v20.6.5.8-stable FIXME as compared to v20.6.4.44-stable + +#### Performance Improvement +* Backported in [#14189](https://github.com/ClickHouse/ClickHouse/issues/14189): Slightly optimize very short queries with LowCardinality. [#14129](https://github.com/ClickHouse/ClickHouse/pull/14129) ([Anton Popov](https://github.com/CurtizJ)). + +#### Improvement +* Backported in [#13950](https://github.com/ClickHouse/ClickHouse/issues/13950): Fix wrong error for long queries. It was possible to get syntax error other than `Max query size exceeded` for correct query. [#13928](https://github.com/ClickHouse/ClickHouse/pull/13928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Bug Fix +* Backported in [#14090](https://github.com/ClickHouse/ClickHouse/issues/14090): Removed wrong auth access check when using ClickHouseDictionarySource to query remote tables. [#12756](https://github.com/ClickHouse/ClickHouse/pull/12756) ([sundyli](https://github.com/sundy-li)). +* Backported in [#13610](https://github.com/ClickHouse/ClickHouse/issues/13610): Fix missing or excessive headers in `TSV/CSVWithNames` formats. This fixes [#12504](https://github.com/ClickHouse/ClickHouse/issues/12504). [#13343](https://github.com/ClickHouse/ClickHouse/pull/13343) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13413](https://github.com/ClickHouse/ClickHouse/issues/13413): Fixed the deadlock in textlog. It is a part of [#12339](https://github.com/ClickHouse/ClickHouse/issues/12339). This fixes [#12325](https://github.com/ClickHouse/ClickHouse/issues/12325). [#13386](https://github.com/ClickHouse/ClickHouse/pull/13386) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#13537](https://github.com/ClickHouse/ClickHouse/issues/13537): Fix wrong code in function `netloc`. This fixes [#13335](https://github.com/ClickHouse/ClickHouse/issues/13335). [#13446](https://github.com/ClickHouse/ClickHouse/pull/13446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#14016](https://github.com/ClickHouse/ClickHouse/issues/14016): Fix premature `ON CLUSTER` timeouts for queries that must be executed on a single replica. Fixes [#6704](https://github.com/ClickHouse/ClickHouse/issues/6704), Fixes [#7228](https://github.com/ClickHouse/ClickHouse/issues/7228), Fixes [#13361](https://github.com/ClickHouse/ClickHouse/issues/13361), Fixes [#11884](https://github.com/ClickHouse/ClickHouse/issues/11884). [#13450](https://github.com/ClickHouse/ClickHouse/pull/13450) ([alesapin](https://github.com/alesapin)). +* Backported in [#14000](https://github.com/ClickHouse/ClickHouse/issues/14000): Fixed the behaviour when sometimes cache-dictionary returned default value instead of present value from source. [#13624](https://github.com/ClickHouse/ClickHouse/pull/13624) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14108](https://github.com/ClickHouse/ClickHouse/issues/14108): Fix visible data clobbering by progress bar in client in interactive mode. This fixes [#12562](https://github.com/ClickHouse/ClickHouse/issues/12562) and [#13369](https://github.com/ClickHouse/ClickHouse/issues/13369) and [#13584](https://github.com/ClickHouse/ClickHouse/issues/13584) and fixes [#12964](https://github.com/ClickHouse/ClickHouse/issues/12964). [#13691](https://github.com/ClickHouse/ClickHouse/pull/13691) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#14081](https://github.com/ClickHouse/ClickHouse/issues/14081): Fixes /replicas_status endpoint response status code when verbose=1. [#13722](https://github.com/ClickHouse/ClickHouse/pull/13722) ([javi santana](https://github.com/javisantana)). +* Backported in [#13776](https://github.com/ClickHouse/ClickHouse/issues/13776): Fix logging Settings.Names/Values when log_queries_min_type > QUERY_START. [#13737](https://github.com/ClickHouse/ClickHouse/pull/13737) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13772](https://github.com/ClickHouse/ClickHouse/issues/13772): Fix race condition between DETACH and background merges. Parts may revive after detach. This is continuation of [#8602](https://github.com/ClickHouse/ClickHouse/issues/8602) that did not fix the issue but introduced a test that started to fail in very rare cases, demonstrating the issue. [#13746](https://github.com/ClickHouse/ClickHouse/pull/13746) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13788](https://github.com/ClickHouse/ClickHouse/issues/13788): Add range check for h3KRing function. This fixes [#13633](https://github.com/ClickHouse/ClickHouse/issues/13633). [#13752](https://github.com/ClickHouse/ClickHouse/pull/13752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13801](https://github.com/ClickHouse/ClickHouse/issues/13801): Fix step overflow in range(). [#13790](https://github.com/ClickHouse/ClickHouse/pull/13790) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13810](https://github.com/ClickHouse/ClickHouse/issues/13810): Fix reading from MergeTree table with INDEX of type SET fails when comparing against NULL. This fixes [#13686](https://github.com/ClickHouse/ClickHouse/issues/13686). [#13793](https://github.com/ClickHouse/ClickHouse/pull/13793) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#13833](https://github.com/ClickHouse/ClickHouse/issues/13833): Fix topK/topKWeighted merge (with non-default parameters). [#13817](https://github.com/ClickHouse/ClickHouse/pull/13817) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13974](https://github.com/ClickHouse/ClickHouse/issues/13974): Fixed potential deadlock when renaming `Distributed` table. [#13922](https://github.com/ClickHouse/ClickHouse/pull/13922) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#14061](https://github.com/ClickHouse/ClickHouse/issues/14061): Fix wrong results in select queries with `DISTINCT` keyword in case `optimize_duplicate_order_by_and_distinct` setting is enabled. [#13925](https://github.com/ClickHouse/ClickHouse/pull/13925) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#14076](https://github.com/ClickHouse/ClickHouse/issues/14076): Fixed wrong mount point in extra info for `Poco::Exception: no space left on device`. [#14050](https://github.com/ClickHouse/ClickHouse/pull/14050) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#14138](https://github.com/ClickHouse/ClickHouse/issues/14138): Fix pointInPolygon with const 2d array as polygon. [#14079](https://github.com/ClickHouse/ClickHouse/pull/14079) ([Alexey Ilyukhov](https://github.com/livace)). +* Backported in [#14172](https://github.com/ClickHouse/ClickHouse/issues/14172): Fix creation of tables with named tuples. This fixes [#13027](https://github.com/ClickHouse/ClickHouse/issues/13027). [#14143](https://github.com/ClickHouse/ClickHouse/pull/14143) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#14294](https://github.com/ClickHouse/ClickHouse/issues/14294): Fix crash when INERT INTO Kafka engine table from an empty result set with a subquery. ... [#14203](https://github.com/ClickHouse/ClickHouse/pull/14203) ([Dongdong Yang](https://github.com/donge)). +* Backported in [#14243](https://github.com/ClickHouse/ClickHouse/issues/14243): Fixed incorrect sorting order if LowCardinality column. This fixes [#13958](https://github.com/ClickHouse/ClickHouse/issues/13958). [#14223](https://github.com/ClickHouse/ClickHouse/pull/14223) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14306](https://github.com/ClickHouse/ClickHouse/issues/14306): Fix segfault in `clickhouse-odbc-bridge` during schema fetch from some external sources. This PR fixes [#13861](https://github.com/ClickHouse/ClickHouse/issues/13861). [#14267](https://github.com/ClickHouse/ClickHouse/pull/14267) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#14340](https://github.com/ClickHouse/ClickHouse/issues/14340): Fix crash during `ALTER` query for table which was created `AS table_function`. Fixes [#14212](https://github.com/ClickHouse/ClickHouse/issues/14212). [#14326](https://github.com/ClickHouse/ClickHouse/pull/14326) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v20.6.6.7-stable.md b/docs/changelogs/v20.6.6.7-stable.md new file mode 100644 index 00000000000..5dfc65eed36 --- /dev/null +++ b/docs/changelogs/v20.6.6.7-stable.md @@ -0,0 +1,13 @@ +### ClickHouse release v20.6.6.7-stable FIXME as compared to v20.6.5.8-stable + +#### Improvement +* Backported in [#14359](https://github.com/ClickHouse/ClickHouse/issues/14359): Added Redis requirepass authorization. [#13688](https://github.com/ClickHouse/ClickHouse/pull/13688) ([Ivan Torgashov](https://github.com/it1804)). + +#### Bug Fix +* Backported in [#14365](https://github.com/ClickHouse/ClickHouse/issues/14365): Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. [#14334](https://github.com/ClickHouse/ClickHouse/pull/14334) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#14540](https://github.com/ClickHouse/ClickHouse/issues/14540): Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. Continuation of [#14334](https://github.com/ClickHouse/ClickHouse/issues/14334). [#14402](https://github.com/ClickHouse/ClickHouse/pull/14402) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#14488](https://github.com/ClickHouse/ClickHouse/issues/14488): Fix bug which leads to wrong merges assignment if table has partitions with a single part. [#14444](https://github.com/ClickHouse/ClickHouse/pull/14444) ([alesapin](https://github.com/alesapin)). +* Backported in [#14483](https://github.com/ClickHouse/ClickHouse/issues/14483): Check for array size overflow in `topK` aggregate function. Without this check the user may send a query with carefully crafter parameters that will lead to server crash. This closes [#14452](https://github.com/ClickHouse/ClickHouse/issues/14452). [#14467](https://github.com/ClickHouse/ClickHouse/pull/14467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#14598](https://github.com/ClickHouse/ClickHouse/issues/14598): Fix rare segfaults in functions with combinator -Resample, which could appear in result of overflow with very large parameters. [#14562](https://github.com/ClickHouse/ClickHouse/pull/14562) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#14667](https://github.com/ClickHouse/ClickHouse/issues/14667): Fix wrong Decimal multiplication result caused wrong decimal scale of result column. [#14603](https://github.com/ClickHouse/ClickHouse/pull/14603) ([Artem Zuikov](https://github.com/4ertus2)). + diff --git a/docs/changelogs/v20.6.7.4-stable.md b/docs/changelogs/v20.6.7.4-stable.md new file mode 100644 index 00000000000..f1ad463ae0d --- /dev/null +++ b/docs/changelogs/v20.6.7.4-stable.md @@ -0,0 +1,9 @@ +### ClickHouse release v20.6.7.4-stable FIXME as compared to v20.6.6.7-stable + +#### Bug Fix +* Backported in [#14726](https://github.com/ClickHouse/ClickHouse/issues/14726): Cleanup data directory after Zookeeper exceptions during CreateQuery for StorageReplicatedMergeTree Engine. [#14563](https://github.com/ClickHouse/ClickHouse/pull/14563) ([Bharat Nallan](https://github.com/bharatnc)). +* Backported in [#14805](https://github.com/ClickHouse/ClickHouse/issues/14805): Fix bug when `ALTER UPDATE` mutation with Nullable column in assignment expression and constant value (like `UPDATE x = 42`) leads to incorrect value in column or segfault. Fixes [#13634](https://github.com/ClickHouse/ClickHouse/issues/13634), [#14045](https://github.com/ClickHouse/ClickHouse/issues/14045). [#14646](https://github.com/ClickHouse/ClickHouse/pull/14646) ([alesapin](https://github.com/alesapin)). +* Backported in [#14723](https://github.com/ClickHouse/ClickHouse/issues/14723): Fixed missed default database name in metadata of materialized view when executing `ALTER ... MODIFY QUERY`. [#14664](https://github.com/ClickHouse/ClickHouse/pull/14664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#14913](https://github.com/ClickHouse/ClickHouse/issues/14913): Fix SIGSEGV for an attempt to INSERT into StorageFile(fd). [#14887](https://github.com/ClickHouse/ClickHouse/pull/14887) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#14945](https://github.com/ClickHouse/ClickHouse/issues/14945): Fix the issue when some invocations of `extractAllGroups` function may trigger "Memory limit exceeded" error. This fixes [#13383](https://github.com/ClickHouse/ClickHouse/issues/13383). [#14889](https://github.com/ClickHouse/ClickHouse/pull/14889) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.6.8.5-stable.md b/docs/changelogs/v20.6.8.5-stable.md new file mode 100644 index 00000000000..530e4958d34 --- /dev/null +++ b/docs/changelogs/v20.6.8.5-stable.md @@ -0,0 +1,32 @@ +### ClickHouse release v20.6.8.5-stable FIXME as compared to v20.6.7.4-stable + +#### Improvement +* Backported in [#15567](https://github.com/ClickHouse/ClickHouse/issues/15567): Now it's possible to change the type of version column for `VersionedCollapsingMergeTree` with `ALTER` query. [#15442](https://github.com/ClickHouse/ClickHouse/pull/15442) ([alesapin](https://github.com/alesapin)). + +#### Bug Fix +* Backported in [#15018](https://github.com/ClickHouse/ClickHouse/issues/15018): Fixed the incorrect sorting order of `Nullable` column. This fixes [#14344](https://github.com/ClickHouse/ClickHouse/issues/14344). [#14495](https://github.com/ClickHouse/ClickHouse/pull/14495) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14826](https://github.com/ClickHouse/ClickHouse/issues/14826): Fix wrong monotonicity detection for shrunk `Int -> Int` cast of signed types. It might lead to incorrect query result. This bug is unveiled in [#14513](https://github.com/ClickHouse/ClickHouse/issues/14513). [#14783](https://github.com/ClickHouse/ClickHouse/pull/14783) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15147](https://github.com/ClickHouse/ClickHouse/issues/15147): Fix a problem where the server may get stuck on startup while talking to ZooKeeper, if the configuration files have to be fetched from ZK (using the `from_zk` include option). This fixes [#14814](https://github.com/ClickHouse/ClickHouse/issues/14814). [#14843](https://github.com/ClickHouse/ClickHouse/pull/14843) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#15253](https://github.com/ClickHouse/ClickHouse/issues/15253): Fixed segfault in CacheDictionary [#14837](https://github.com/ClickHouse/ClickHouse/issues/14837). [#14879](https://github.com/ClickHouse/ClickHouse/pull/14879) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14986](https://github.com/ClickHouse/ClickHouse/issues/14986): Publish CPU frequencies per logical core in `system.asynchronous_metrics`. This fixes [#14923](https://github.com/ClickHouse/ClickHouse/issues/14923). [#14924](https://github.com/ClickHouse/ClickHouse/pull/14924) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#14968](https://github.com/ClickHouse/ClickHouse/issues/14968): Fix to make predicate push down work when subquery contains finalizeAggregation function. Fixes [#14847](https://github.com/ClickHouse/ClickHouse/issues/14847). [#14937](https://github.com/ClickHouse/ClickHouse/pull/14937) ([filimonov](https://github.com/filimonov)). +* Backported in [#15080](https://github.com/ClickHouse/ClickHouse/issues/15080): Fix crash in RIGHT or FULL JOIN with join_algorith='auto' when memory limit exceeded and we should change HashJoin with MergeJoin. [#15002](https://github.com/ClickHouse/ClickHouse/pull/15002) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15057](https://github.com/ClickHouse/ClickHouse/issues/15057): If function `bar` was called with specifically crafted arguments, buffer overflow was possible. This closes [#13926](https://github.com/ClickHouse/ClickHouse/issues/13926). [#15028](https://github.com/ClickHouse/ClickHouse/pull/15028) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15050](https://github.com/ClickHouse/ClickHouse/issues/15050): We already use padded comparison between String and FixedString (https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionsComparison.h#L333). This PR applies the same logic to field comparison which corrects the usage of FixedString as primary keys. This fixes [#14908](https://github.com/ClickHouse/ClickHouse/issues/14908). [#15033](https://github.com/ClickHouse/ClickHouse/pull/15033) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15143](https://github.com/ClickHouse/ClickHouse/issues/15143): Fixes `Data compressed with different methods` in `join_algorithm='auto'`. Keep LowCardinality as type for left table join key in `join_algorithm='partial_merge'`. [#15088](https://github.com/ClickHouse/ClickHouse/pull/15088) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15223](https://github.com/ClickHouse/ClickHouse/issues/15223): Fix bug in table engine `Buffer` which doesn't allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). +* Backported in [#15404](https://github.com/ClickHouse/ClickHouse/issues/15404): Fix instance crash when using joinGet with LowCardinality types. This fixes [#15214](https://github.com/ClickHouse/ClickHouse/issues/15214). [#15220](https://github.com/ClickHouse/ClickHouse/pull/15220) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15279](https://github.com/ClickHouse/ClickHouse/issues/15279): Fix MSan report in QueryLog. Uninitialized memory can be used for the field `memory_usage`. [#15258](https://github.com/ClickHouse/ClickHouse/pull/15258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15412](https://github.com/ClickHouse/ClickHouse/issues/15412): Fix hang of queries with a lot of subqueries to same table of `MySQL` engine. Previously, if there were more than 16 subqueries to same `MySQL` table in query, it hang forever. [#15299](https://github.com/ClickHouse/ClickHouse/pull/15299) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#15339](https://github.com/ClickHouse/ClickHouse/issues/15339): Fix rare race condition on server startup when system.logs are enabled. [#15300](https://github.com/ClickHouse/ClickHouse/pull/15300) ([alesapin](https://github.com/alesapin)). +* Backported in [#15335](https://github.com/ClickHouse/ClickHouse/issues/15335): Fix race condition during MergeTree table rename and background cleanup. [#15304](https://github.com/ClickHouse/ClickHouse/pull/15304) ([alesapin](https://github.com/alesapin)). +* Backported in [#15445](https://github.com/ClickHouse/ClickHouse/issues/15445): Report proper error when the second argument of `boundingRatio` aggregate function has a wrong type. [#15407](https://github.com/ClickHouse/ClickHouse/pull/15407) ([detailyang](https://github.com/detailyang)). +* Backported in [#15508](https://github.com/ClickHouse/ClickHouse/issues/15508): Fix bug with event subscription in DDLWorker which rarely may lead to query hangs in `ON CLUSTER`. Introduced in [#13450](https://github.com/ClickHouse/ClickHouse/issues/13450). [#15477](https://github.com/ClickHouse/ClickHouse/pull/15477) ([alesapin](https://github.com/alesapin)). +* Backported in [#15616](https://github.com/ClickHouse/ClickHouse/issues/15616): Fix `Missing columns` errors when selecting columns which absent in data, but depend on other columns which also absent in data. Fixes [#15530](https://github.com/ClickHouse/ClickHouse/issues/15530). [#15532](https://github.com/ClickHouse/ClickHouse/pull/15532) ([alesapin](https://github.com/alesapin)). +* Backported in [#15561](https://github.com/ClickHouse/ClickHouse/issues/15561): Fix bug when `ILIKE` operator stops being case insensitive if `LIKE` with the same pattern was executed. [#15536](https://github.com/ClickHouse/ClickHouse/pull/15536) ([alesapin](https://github.com/alesapin)). +* Backported in [#15728](https://github.com/ClickHouse/ClickHouse/issues/15728): Mutation might hang waiting for some non-existent part after `MOVE` or `REPLACE PARTITION` or, in rare cases, after `DETACH` or `DROP PARTITION`. It's fixed. [#15537](https://github.com/ClickHouse/ClickHouse/pull/15537) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15654](https://github.com/ClickHouse/ClickHouse/issues/15654): Fix 'Database doesn't exist.' in queries with IN and Distributed table when there's no database on initiator. [#15538](https://github.com/ClickHouse/ClickHouse/pull/15538) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15586](https://github.com/ClickHouse/ClickHouse/issues/15586): Prevent the possibility of error message `Could not calculate available disk space (statvfs), errno: 4, strerror: Interrupted system call`. This fixes [#15541](https://github.com/ClickHouse/ClickHouse/issues/15541). [#15557](https://github.com/ClickHouse/ClickHouse/pull/15557) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15664](https://github.com/ClickHouse/ClickHouse/issues/15664): Fixed `Element ... is not a constant expression` error when using `JSON*` function result in `VALUES`, `LIMIT` or right side of `IN` operator. [#15589](https://github.com/ClickHouse/ClickHouse/pull/15589) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15714](https://github.com/ClickHouse/ClickHouse/issues/15714): Fix the order of destruction for resources in `ReadFromStorage` step of query plan. It might cause crashes in rare cases. Possibly connected with [#15610](https://github.com/ClickHouse/ClickHouse/issues/15610). [#15645](https://github.com/ClickHouse/ClickHouse/pull/15645) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v20.6.9.1-stable.md b/docs/changelogs/v20.6.9.1-stable.md new file mode 100644 index 00000000000..f5860bcb3ee --- /dev/null +++ b/docs/changelogs/v20.6.9.1-stable.md @@ -0,0 +1,2 @@ +### ClickHouse release v20.6.9.1-stable FIXME as compared to v20.6.8.5-stable + diff --git a/docs/changelogs/v20.7.1.4310-prestable.md b/docs/changelogs/v20.7.1.4310-prestable.md new file mode 100644 index 00000000000..a941e071fc6 --- /dev/null +++ b/docs/changelogs/v20.7.1.4310-prestable.md @@ -0,0 +1,170 @@ +### ClickHouse release v20.7.1.4310-prestable FIXME as compared to v20.6.1.4066-prestable + +#### Backward Incompatible Change +* Deprecate special printing of zero Date/DateTime values as `0000-00-00` and `0000-00-00 00:00:00`. [#12442](https://github.com/ClickHouse/ClickHouse/pull/12442) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Function `modulo` (operator `%`) with at least one floating point number as argument will calculate remainder of division directly on floating point numbers without converting both arguments to integers. It makes behaviour compatible with most of DBMS. This also applicable for Date and DateTime data types. Added alias `mod`. This closes [#7323](https://github.com/ClickHouse/ClickHouse/issues/7323). [#12585](https://github.com/ClickHouse/ClickHouse/pull/12585) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* - Added support of LDAP authentication for preconfigured users ("Simple Bind" method). [#11234](https://github.com/ClickHouse/ClickHouse/pull/11234) ([Denis Glazachev](https://github.com/traceon)). +* Add mapAdd and mapSubtract functions for working with tuple maps. [#11735](https://github.com/ClickHouse/ClickHouse/pull/11735) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Added `system.crash_log` table into which stack traces for fatal errors are collected. [#12316](https://github.com/ClickHouse/ClickHouse/pull/12316) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `bayesAB` function for bayesian-ab-testing. [#12327](https://github.com/ClickHouse/ClickHouse/pull/12327) ([achimbab](https://github.com/achimbab)). +* Integration with [COS](https://intl.cloud.tencent.com/product/cos). [#12386](https://github.com/ClickHouse/ClickHouse/pull/12386) ([fastio](https://github.com/fastio)). +* - Add FROM_UNIXTIME function, related to [12149](https://github.com/ClickHouse/ClickHouse/issues/12149). [#12484](https://github.com/ClickHouse/ClickHouse/pull/12484) ([flynn](https://github.com/ucasfl)). +* A function `formatRow` is added to support turning arbitrary expressions into a string via given format. It's useful for manipulating SQL outputs and is quite versatile combined with the `columns` function. [#12574](https://github.com/ClickHouse/ClickHouse/pull/12574) ([Amos Bird](https://github.com/amosbird)). +* Add setting `allow_non_metadata_alters` which restricts to execute `ALTER` queries which modify data on disk. Disabled be default. Closes [#11547](https://github.com/ClickHouse/ClickHouse/issues/11547). [#12635](https://github.com/ClickHouse/ClickHouse/pull/12635) ([alesapin](https://github.com/alesapin)). +* Add `minMap` and `maxMap` functions support to `SimpleAggregateFunction`. [#12662](https://github.com/ClickHouse/ClickHouse/pull/12662) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Added http headers `X-ClickHouse-Database` and `X-ClickHouse-Format` which may be used to set default database and output format. [#12981](https://github.com/ClickHouse/ClickHouse/pull/12981) ([hcz](https://github.com/hczhcz)). +* Implement user-defined settings. [#13013](https://github.com/ClickHouse/ClickHouse/pull/13013) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Performance Improvement +* Optimize PK lookup for queries that match exact PK range. [#12277](https://github.com/ClickHouse/ClickHouse/pull/12277) ([Ivan Babrou](https://github.com/bobrik)). +* Attempt to implement streaming optimization in `DiskS3`. [#12434](https://github.com/ClickHouse/ClickHouse/pull/12434) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Replaces monotonous functions with its argument in ORDER BY if `set optimize_monotonous_functions_in_order_by=1`. [#12467](https://github.com/ClickHouse/ClickHouse/pull/12467) ([Artem Zuikov](https://github.com/4ertus2)). +* Converts String-type arguments of function "if" and "transform" into enum if `set optimize_if_transform_strings_to_enum = 1`. [#12515](https://github.com/ClickHouse/ClickHouse/pull/12515) ([Artem Zuikov](https://github.com/4ertus2)). +* Parallel PK lookup and skipping index stages on parts, as described in [#11564](https://github.com/ClickHouse/ClickHouse/issues/11564). [#12589](https://github.com/ClickHouse/ClickHouse/pull/12589) ([Ivan Babrou](https://github.com/bobrik)). +* Push down `LIMIT` step for query plan. [#13016](https://github.com/ClickHouse/ClickHouse/pull/13016) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Slightly improve performance of aggregation by UInt8/UInt16 keys. [#13055](https://github.com/ClickHouse/ClickHouse/pull/13055) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Slightly improve performance of aggregation by UInt8/UInt16 keys. [#13091](https://github.com/ClickHouse/ClickHouse/pull/13091) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Adds a new type of polygon dictionary which uses a recursively built grid to reduce the number of polygons which need to be checked for each point. [#9278](https://github.com/ClickHouse/ClickHouse/pull/9278) ([achulkov2](https://github.com/achulkov2)). +* Allow TabSeparatedRaw as an input format. [#12009](https://github.com/ClickHouse/ClickHouse/pull/12009) ([hcz](https://github.com/hczhcz)). +* Separated `AWSAuthV4Signer` into different logger, removed "AWSClient: AWSClient". [#12320](https://github.com/ClickHouse/ClickHouse/pull/12320) ([Vladimir Chebotarev](https://github.com/excitoon)). +* - Implement `RENAME DATABASE` and `RENAME DICTIONARY` for `Atomic` database engine - Add implicit `{uuid}` macro, which can be used in ZooKeeper path for `ReplicatedMergeTree`. It works with `CREATE ... ON CLUSTER ...` queries. Set `show_table_uuid_in_table_create_query_if_not_nil` to `true` to use it. - Make `ReplicatedMergeTree` engine arguments optional, `/clickhouse/tables/{uuid}/{shard}/` and `{replica}` are used by default. Closes [#12135](https://github.com/ClickHouse/ClickHouse/issues/12135). - Minor fixes. - These changes break backward compatibility of `Atomic` database engine. Previously created `Atomic` databases must be manually converted to new format. [#12343](https://github.com/ClickHouse/ClickHouse/pull/12343) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Now joinGet supports multi-key lookup. [#12418](https://github.com/ClickHouse/ClickHouse/pull/12418) ([Amos Bird](https://github.com/amosbird)). +* Rollback insertion errors in `Log`, `TinyLog`, `StripeLog` engines. In previous versions insertion error lead to inconsisent table state (this works as documented and it is normal for these table engines). This fixes [#12402](https://github.com/ClickHouse/ClickHouse/issues/12402). [#12426](https://github.com/ClickHouse/ClickHouse/pull/12426) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid overflow in parsing of DateTime values that will lead to negative unix timestamp in their timezone (for example, `1970-01-01 00:00:00` in Moscow). Saturate to zero instead. This fixes [#3470](https://github.com/ClickHouse/ClickHouse/issues/3470). This fixes [#4172](https://github.com/ClickHouse/ClickHouse/issues/4172). [#12443](https://github.com/ClickHouse/ClickHouse/pull/12443) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add several metrics for requests to S3 storages. [#12464](https://github.com/ClickHouse/ClickHouse/pull/12464) ([ianton-ru](https://github.com/ianton-ru)). +* Changes default value for `multiple_joins_rewriter_version` to 2. It enables new multiple joins rewriter that knows about column names. [#12469](https://github.com/ClickHouse/ClickHouse/pull/12469) ([Artem Zuikov](https://github.com/4ertus2)). +* Allow to set JOIN kind and type in more standad way: `LEFT SEMI JOIN` instead of `SEMI LEFT JOIN`. For now both are correct. [#12520](https://github.com/ClickHouse/ClickHouse/pull/12520) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix explain query format overwrite by default, issue [#12432](https://github.com/ClickHouse/ClickHouse/issues/12432). [#12541](https://github.com/ClickHouse/ClickHouse/pull/12541) ([BohuTANG](https://github.com/BohuTANG)). +* Add SelectedRows and SelectedBytes events. [#12638](https://github.com/ClickHouse/ClickHouse/pull/12638) ([ianton-ru](https://github.com/ianton-ru)). +* Add `current_database` information to `system.query_log`. [#12652](https://github.com/ClickHouse/ClickHouse/pull/12652) ([Amos Bird](https://github.com/amosbird)). +* Support truncate table without table keyword. [#12653](https://github.com/ClickHouse/ClickHouse/pull/12653) ([Winter Zhang](https://github.com/zhang2014)). +* Now exceptions forwarded to the client if an error happened during ALTER or mutation. Closes [#11329](https://github.com/ClickHouse/ClickHouse/issues/11329). [#12666](https://github.com/ClickHouse/ClickHouse/pull/12666) ([alesapin](https://github.com/alesapin)). +* Protect from the cases when user may set `background_pool_size` to value lower than `number_of_free_entries_in_pool_to_execute_mutation` or `number_of_free_entries_in_pool_to_lower_max_size_of_merge`. In these cases ALTERs won't work or the maximum size of merge will be too limited. Saturate values instead. This closes [#10897](https://github.com/ClickHouse/ClickHouse/issues/10897). [#12728](https://github.com/ClickHouse/ClickHouse/pull/12728) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* This change makes skipping index efficiency more obvious by showing both skipped and total examined granules. [#12754](https://github.com/ClickHouse/ClickHouse/pull/12754) ([Ivan Babrou](https://github.com/bobrik)). +* This change makes skipping index efficiency more obvious by showing total marks before and after skipping indices. [#12755](https://github.com/ClickHouse/ClickHouse/pull/12755) ([Ivan Babrou](https://github.com/bobrik)). +* Introduce setting `alter_partition_verbose_result` which outputs information about touched parts for some types of `ALTER TABLE ... PARTITION ...` queries (currently `ATTACH` and `FREEZE`). Closes [#8076](https://github.com/ClickHouse/ClickHouse/issues/8076). [#13017](https://github.com/ClickHouse/ClickHouse/pull/13017) ([alesapin](https://github.com/alesapin)). +* Add QueryTimeMicroseconds, SelectQueryTimeMicroseconds and InsertQueryTimeMicroseconds to system.events. [#13028](https://github.com/ClickHouse/ClickHouse/pull/13028) ([ianton-ru](https://github.com/ianton-ru)). +* break-ing out of a loop because it makes sense to do so. [#13058](https://github.com/ClickHouse/ClickHouse/pull/13058) ([Mark Papadakis](https://github.com/markpapadakis)). +* Keep less amount of logs in ZooKeeper. Avoid too large growth of ZooKeeper nodes in case of offline replicas when having many servers/tables/inserts. [#13100](https://github.com/ClickHouse/ClickHouse/pull/13100) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add sanity check for MergeTree settings. If the settings are incorrect, the server will refuse to start or to create a table, printing detailed explanation to the user. [#13153](https://github.com/ClickHouse/ClickHouse/pull/13153) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow *Map aggregate functions to work on Arrays with NULLs. Fixes [#13157](https://github.com/ClickHouse/ClickHouse/issues/13157). [#13225](https://github.com/ClickHouse/ClickHouse/pull/13225) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix assert in parseDateTimeBestEffort. This fixes [#12649](https://github.com/ClickHouse/ClickHouse/issues/12649). [#13227](https://github.com/ClickHouse/ClickHouse/pull/13227) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix assert in geohashesInBox. This fixes [#12554](https://github.com/ClickHouse/ClickHouse/issues/12554). [#13229](https://github.com/ClickHouse/ClickHouse/pull/13229) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now broken parts are also reported when encountered in compact part processing. [#13282](https://github.com/ClickHouse/ClickHouse/pull/13282) ([Amos Bird](https://github.com/amosbird)). +* Fix a 'Week'-interval formatting for ATTACH/ALTER/CREATE QUOTA-statements. [#13417](https://github.com/ClickHouse/ClickHouse/pull/13417) ([vladimir-golovchenko](https://github.com/vladimir-golovchenko)). +* Updated gitignore-files. [#13447](https://github.com/ClickHouse/ClickHouse/pull/13447) ([vladimir-golovchenko](https://github.com/vladimir-golovchenko)). + +#### Bug Fix +* kafka: fix SIGSEGV if there is an message with error in the middle of the batch. [#12302](https://github.com/ClickHouse/ClickHouse/pull/12302) ([Azat Khuzhin](https://github.com/azat)). +* If MergeTree table does not contain ORDER BY or PARTITION BY, it was possible to request ALTER to CLEAR all the columns and ALTER will stuck. Fixed [#7941](https://github.com/ClickHouse/ClickHouse/issues/7941). [#12382](https://github.com/ClickHouse/ClickHouse/pull/12382) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* SystemLog: do not write to ordinary server log under mutex. This can lead to deadlock if `text_log` is enabled. [#12452](https://github.com/ClickHouse/ClickHouse/pull/12452) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix crash in JOIN with dictionary when we are joining over expression of dictionary key: `t JOIN dict ON expr(dict.id) = t.id`. Disable dictionary join optimisation for this case. [#12458](https://github.com/ClickHouse/ClickHouse/pull/12458) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix SETTINGS parse after FORMAT. [#12480](https://github.com/ClickHouse/ClickHouse/pull/12480) ([Azat Khuzhin](https://github.com/azat)). +* Fix backwards compatibility in binary format of `AggregateFunction(avg, ...)` values. This fixes [#12342](https://github.com/ClickHouse/ClickHouse/issues/12342). [#12486](https://github.com/ClickHouse/ClickHouse/pull/12486) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed performance issue, while reading from compact parts. [#12492](https://github.com/ClickHouse/ClickHouse/pull/12492) ([Anton Popov](https://github.com/CurtizJ)). +* Fixing race condition in live view tables which could cause data duplication. [#12519](https://github.com/ClickHouse/ClickHouse/pull/12519) ([vzakaznikov](https://github.com/vzakaznikov)). +* Better exception for function `in` with invalid number of arguments. [#12529](https://github.com/ClickHouse/ClickHouse/pull/12529) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bug which lead to broken old parts after `ALTER DELETE` query when `enable_mixed_granularity_parts=1`. Fixes [#12536](https://github.com/ClickHouse/ClickHouse/issues/12536). [#12543](https://github.com/ClickHouse/ClickHouse/pull/12543) ([alesapin](https://github.com/alesapin)). +* Now ClickHouse will recalculate checksums for parts when file `checksums.txt` is absent. Broken since [#9827](https://github.com/ClickHouse/ClickHouse/issues/9827). [#12545](https://github.com/ClickHouse/ClickHouse/pull/12545) ([alesapin](https://github.com/alesapin)). +* Remove data for Distributed tables (blocks from async INSERTs) on DROP TABLE. [#12556](https://github.com/ClickHouse/ClickHouse/pull/12556) ([Azat Khuzhin](https://github.com/azat)). +* Fix race condition in external dictionaries with cache layout which can lead server crash. [#12566](https://github.com/ClickHouse/ClickHouse/pull/12566) ([alesapin](https://github.com/alesapin)). +* Fix lack of aliases with function `any`. [#12593](https://github.com/ClickHouse/ClickHouse/pull/12593) ([Anton Popov](https://github.com/CurtizJ)). +* Fix error `Output of TreeExecutor is not sorted` for `OPTIMIZE DEDUPLICATE`. Fixes [#11572](https://github.com/ClickHouse/ClickHouse/issues/11572). [#12613](https://github.com/ClickHouse/ClickHouse/pull/12613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible `Pipeline stuck` error for queries with external sorting. Fixes [#12617](https://github.com/ClickHouse/ClickHouse/issues/12617). [#12618](https://github.com/ClickHouse/ClickHouse/pull/12618) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* The function `groupArrayMoving*` was not working for distributed queries. It's result was calculated within incorrect data type (without promotion to the largest type). The function `groupArrayMovingAvg` was returning integer number that was inconsistent with the `avg` function. This fixes [#12568](https://github.com/ClickHouse/ClickHouse/issues/12568). [#12622](https://github.com/ClickHouse/ClickHouse/pull/12622) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error message about adaptive granularity. [#12624](https://github.com/ClickHouse/ClickHouse/pull/12624) ([alesapin](https://github.com/alesapin)). +* Better exception message in disk access storage. [#12625](https://github.com/ClickHouse/ClickHouse/pull/12625) ([alesapin](https://github.com/alesapin)). +* Exception `There is no supertype...` can be thrown during `ALTER ... UPDATE` in unexpected cases (e.g. when subtracting from UInt64 column). This fixes [#7306](https://github.com/ClickHouse/ClickHouse/issues/7306). This fixes [#4165](https://github.com/ClickHouse/ClickHouse/issues/4165). [#12633](https://github.com/ClickHouse/ClickHouse/pull/12633) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* CREATE USER IF NOT EXISTS now doesn't throw exception if the user exists. This fixes [#12507](https://github.com/ClickHouse/ClickHouse/issues/12507). [#12646](https://github.com/ClickHouse/ClickHouse/pull/12646) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add support for function `if` with `Array(UUID)` arguments. This fixes [#11066](https://github.com/ClickHouse/ClickHouse/issues/11066). [#12648](https://github.com/ClickHouse/ClickHouse/pull/12648) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix SIGSEGV in StorageKafka when broker is unavailable (and not only). [#12658](https://github.com/ClickHouse/ClickHouse/pull/12658) ([Azat Khuzhin](https://github.com/azat)). +* fixes [#10572](https://github.com/ClickHouse/ClickHouse/issues/10572) fix bloom filter index with const expression. [#12659](https://github.com/ClickHouse/ClickHouse/pull/12659) ([Winter Zhang](https://github.com/zhang2014)). +* fixes [#12293](https://github.com/ClickHouse/ClickHouse/issues/12293) allow push predicate when subquery contains with clause. [#12663](https://github.com/ClickHouse/ClickHouse/pull/12663) ([Winter Zhang](https://github.com/zhang2014)). +* Fix optimization `optimize_move_functions_out_of_any=1` in case of `any(func())`. [#12664](https://github.com/ClickHouse/ClickHouse/pull/12664) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix memory tracking for input_format_parallel_parsing (by attaching thread to group). [#12672](https://github.com/ClickHouse/ClickHouse/pull/12672) ([Azat Khuzhin](https://github.com/azat)). +* Fix performance with large tuples, which are interpreted as functions in `IN` section. The case when user write `WHERE x IN tuple(1, 2, ...)` instead of `WHERE x IN (1, 2, ...)` for some obscure reason. [#12700](https://github.com/ClickHouse/ClickHouse/pull/12700) ([Anton Popov](https://github.com/CurtizJ)). +* Fix CAST(Nullable(String), Enum()). [#12745](https://github.com/ClickHouse/ClickHouse/pull/12745) ([Azat Khuzhin](https://github.com/azat)). +* Fix rare bug when `ALTER DELETE` and `ALTER MODIFY COLUMN` queries executed simultaneously as a single mutation. Bug leads to an incorrect amount of rows in `count.txt` and as a consequence incorrect data in part. Also, fix a small bug with simultaneous `ALTER RENAME COLUMN` and `ALTER ADD COLUMN`. [#12760](https://github.com/ClickHouse/ClickHouse/pull/12760) ([alesapin](https://github.com/alesapin)). +* Fix unnecessary limiting for the number of threads for selects from local replica. [#12840](https://github.com/ClickHouse/ClickHouse/pull/12840) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix columns duplication for range hashed dictionary created from DDL query. This fixes [#10605](https://github.com/ClickHouse/ClickHouse/issues/10605). [#12857](https://github.com/ClickHouse/ClickHouse/pull/12857) ([alesapin](https://github.com/alesapin)). +* Corrected merge_with_ttl_timeout logic which did not work well when expiration affected more than one partition over one time interval. (Authored by @excitoon). [#12982](https://github.com/ClickHouse/ClickHouse/pull/12982) ([Alexander Kazakov](https://github.com/Akazz)). +* Fix `Block structure mismatch` error for queries with `UNION` and `JOIN`. Fixes [#12602](https://github.com/ClickHouse/ClickHouse/issues/12602). [#12989](https://github.com/ClickHouse/ClickHouse/pull/12989) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash which was possible for queries with `ORDER BY` tuple and small `LIMIT`. Fixes [#12623](https://github.com/ClickHouse/ClickHouse/issues/12623). [#13009](https://github.com/ClickHouse/ClickHouse/pull/13009) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add logging when the mutation is not running because of limited disk space or free threads in the background pool. [#13068](https://github.com/ClickHouse/ClickHouse/pull/13068) ([alesapin](https://github.com/alesapin)). +* Fix error `Cannot convert column because it is constant but values of constants are different in source and result` for remote queries which use deterministic functions in scope of query, but not deterministic between queries, like `now()`, `now64()`, `randConstant()`. Fixes [#11327](https://github.com/ClickHouse/ClickHouse/issues/11327). [#13075](https://github.com/ClickHouse/ClickHouse/pull/13075) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix wrong index analysis with functions. It could lead to pruning wrong parts, while reading from `MergeTree` tables. Fixes [#13060](https://github.com/ClickHouse/ClickHouse/issues/13060). Fixes [#12406](https://github.com/ClickHouse/ClickHouse/issues/12406). [#13081](https://github.com/ClickHouse/ClickHouse/pull/13081) ([Anton Popov](https://github.com/CurtizJ)). +* Fix segfault when mutation killed and the server tries to send the exception to the client. [#13169](https://github.com/ClickHouse/ClickHouse/pull/13169) ([alesapin](https://github.com/alesapin)). +* AvroConfluent: Skip Kafka tombstone records AvroConfluent: Support skipping broken records ... [#13203](https://github.com/ClickHouse/ClickHouse/pull/13203) ([Andrew Onyshchuk](https://github.com/oandrew)). +* Fix DateTime64 conversion functions with constant argument. [#13205](https://github.com/ClickHouse/ClickHouse/pull/13205) ([Azat Khuzhin](https://github.com/azat)). +* Fix assert in `arrayElement` function in case of array elements are Nullable and array subscript is also Nullable. This fixes [#12172](https://github.com/ClickHouse/ClickHouse/issues/12172). [#13224](https://github.com/ClickHouse/ClickHouse/pull/13224) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix function if with nullable constexpr as cond that is not literal NULL. Fixes [#12463](https://github.com/ClickHouse/ClickHouse/issues/12463). [#13226](https://github.com/ClickHouse/ClickHouse/pull/13226) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Return passed number for numbers with MSB set in roundUpToPowerOfTwoOrZero(). [#13234](https://github.com/ClickHouse/ClickHouse/pull/13234) ([Azat Khuzhin](https://github.com/azat)). +* Fix assertion in KeyCondition when primary key contains expression with monotonic function and query contains comparison with constant whose type is different. This fixes [#12465](https://github.com/ClickHouse/ClickHouse/issues/12465). [#13251](https://github.com/ClickHouse/ClickHouse/pull/13251) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potentially low performance and slightly incorrect result for `uniqExact`, `topK`, `sumDistinct` and similar aggregate functions called on Float types with NaN values. It also triggered assert in debug build. This fixes [#12491](https://github.com/ClickHouse/ClickHouse/issues/12491). [#13254](https://github.com/ClickHouse/ClickHouse/pull/13254) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The server may crash if user passed specifically crafted arguments to the function `h3ToChildren`. This fixes [#13275](https://github.com/ClickHouse/ClickHouse/issues/13275). [#13277](https://github.com/ClickHouse/ClickHouse/pull/13277) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix possible error `Totals having transform was already added to pipeline` in case of a query from delayed replica. [#13290](https://github.com/ClickHouse/ClickHouse/pull/13290) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in `LEFT ASOF JOIN` with `join_use_nulls=1`. [#13291](https://github.com/ClickHouse/ClickHouse/pull/13291) ([Artem Zuikov](https://github.com/4ertus2)). +* Throw error on `arrayJoin()` function in `JOIN ON` section. [#13330](https://github.com/ClickHouse/ClickHouse/pull/13330) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix segfault when function `groupArrayMovingSum` deserializes empty state. Fixes [#13339](https://github.com/ClickHouse/ClickHouse/issues/13339). [#13341](https://github.com/ClickHouse/ClickHouse/pull/13341) ([alesapin](https://github.com/alesapin)). +* Fixed the deadlock in textlog. It is a part of [#12339](https://github.com/ClickHouse/ClickHouse/issues/12339). This fixes [#12325](https://github.com/ClickHouse/ClickHouse/issues/12325). [#13386](https://github.com/ClickHouse/ClickHouse/pull/13386) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix PrettyCompactMonoBlock for clickhouse-local. Fix extremes/totals with PrettyCompactMonoBlock. Fixes [#7746](https://github.com/ClickHouse/ClickHouse/issues/7746). [#13394](https://github.com/ClickHouse/ClickHouse/pull/13394) ([Azat Khuzhin](https://github.com/azat)). +* Fix queries with constant columns and `ORDER BY` prefix of primary key. [#13396](https://github.com/ClickHouse/ClickHouse/pull/13396) ([Anton Popov](https://github.com/CurtizJ)). +* Fix empty output for `Arrow` and `Parquet` formats in case if query return zero rows. It was done because empty output is not valid for this formats. [#13399](https://github.com/ClickHouse/ClickHouse/pull/13399) ([hcz](https://github.com/hczhcz)). +* Fix `aggregate function any(x) is found inside another aggregate function in query` error with `SET optimize_move_functions_out_of_any = 1` and aliases inside `any()`. [#13419](https://github.com/ClickHouse/ClickHouse/pull/13419) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix invalid return type for comparison of tuples with `NULL` elements. Fixes [#12461](https://github.com/ClickHouse/ClickHouse/issues/12461). [#13420](https://github.com/ClickHouse/ClickHouse/pull/13420) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix error in `parseDateTimeBestEffort` function when unix timestamp was passed as an argument. This fixes [#13362](https://github.com/ClickHouse/ClickHouse/issues/13362). [#13441](https://github.com/ClickHouse/ClickHouse/pull/13441) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Build/Testing/Packaging Improvement +* Add compiler option to control that stack frames are not too large. This will help to run the code in fibers with small stack size. [#11524](https://github.com/ClickHouse/ClickHouse/pull/11524) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rework integrational configuration paths. [#12285](https://github.com/ClickHouse/ClickHouse/pull/12285) ([Ilya Yatsishin](https://github.com/qoega)). +* Add docker image for fast tests. [#12294](https://github.com/ClickHouse/ClickHouse/pull/12294) ([alesapin](https://github.com/alesapin)). +* Update fmtlib to master (7.0.1). [#12446](https://github.com/ClickHouse/ClickHouse/pull/12446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add PEERDIR(protoc) as protobuf format parses .proto file in runtime. [#12475](https://github.com/ClickHouse/ClickHouse/pull/12475) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Now we check that server is able to start after stress tests run. This fixes [#12473](https://github.com/ClickHouse/ClickHouse/issues/12473). [#12496](https://github.com/ClickHouse/ClickHouse/pull/12496) ([alesapin](https://github.com/alesapin)). +* Improve performance of TestKeeper. This will speedup tests with heavy usage of Replicated tables. [#12505](https://github.com/ClickHouse/ClickHouse/pull/12505) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adding RBAC syntax tests in TestFlows. [#12642](https://github.com/ClickHouse/ClickHouse/pull/12642) ([vzakaznikov](https://github.com/vzakaznikov)). +* Apply random query mutations (fuzzing) in stress tests. [#12734](https://github.com/ClickHouse/ClickHouse/pull/12734) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Connector-ODBC updated to mysql-connector-odbc-8.0.21. [#12739](https://github.com/ClickHouse/ClickHouse/pull/12739) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix build of test under Mac OS X. This closes [#12767](https://github.com/ClickHouse/ClickHouse/issues/12767). [#12772](https://github.com/ClickHouse/ClickHouse/pull/12772) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* increasing timeouts in testflows tests. [#12949](https://github.com/ClickHouse/ClickHouse/pull/12949) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add a test for `ALTER TABLE CLEAR COLUMN` query for primary key columns. [#12951](https://github.com/ClickHouse/ClickHouse/pull/12951) ([alesapin](https://github.com/alesapin)). +* Do not build helper_container image inside integrational tests. Build docker container in CI and use pre-built helper_container in integration tests. [#12953](https://github.com/ClickHouse/ClickHouse/pull/12953) ([Ilya Yatsishin](https://github.com/qoega)). +* Check an ability that we able to restore the backup from an old version to the new version. This closes [#8979](https://github.com/ClickHouse/ClickHouse/issues/8979). [#12959](https://github.com/ClickHouse/ClickHouse/pull/12959) ([alesapin](https://github.com/alesapin)). +* Fix MSan error in "rdkafka" library. This closes [#12990](https://github.com/ClickHouse/ClickHouse/issues/12990). Updated `rdkafka` to version 1.5 (master). [#12991](https://github.com/ClickHouse/ClickHouse/pull/12991) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rerun some tests in fast test check. [#12992](https://github.com/ClickHouse/ClickHouse/pull/12992) ([alesapin](https://github.com/alesapin)). +* Adding extra xfails for some ldap tests. [#13054](https://github.com/ClickHouse/ClickHouse/pull/13054) ([vzakaznikov](https://github.com/vzakaznikov)). +* Added tests for RBAC functionality of `SELECT` privilege in TestFlows. [#13061](https://github.com/ClickHouse/ClickHouse/pull/13061) ([Ritaank Tiwari](https://github.com/ritaank)). +* Rewrote Function tests to gtest. Removed useless includes from tests. [#13073](https://github.com/ClickHouse/ClickHouse/pull/13073) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Improve cache purge in documentation deploy script. [#13107](https://github.com/ClickHouse/ClickHouse/pull/13107) ([alesapin](https://github.com/alesapin)). +* Fixing 00960_live_view_watch_events_live.py test. [#13108](https://github.com/ClickHouse/ClickHouse/pull/13108) ([vzakaznikov](https://github.com/vzakaznikov)). +* Small fixes to the RBAC SRS. [#13152](https://github.com/ClickHouse/ClickHouse/pull/13152) ([vzakaznikov](https://github.com/vzakaznikov)). +* Even more retries in zkutil gtest to prevent test flakiness. [#13165](https://github.com/ClickHouse/ClickHouse/pull/13165) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add script which set labels for pull requests in GitHub hook. [#13183](https://github.com/ClickHouse/ClickHouse/pull/13183) ([alesapin](https://github.com/alesapin)). +* Use `shellcheck` for sh tests linting. [#13200](https://github.com/ClickHouse/ClickHouse/pull/13200) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use `shellcheck` for sh tests linting. [#13207](https://github.com/ClickHouse/ClickHouse/pull/13207) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix libunwind build in AArch64. This fixes [#13204](https://github.com/ClickHouse/ClickHouse/issues/13204). [#13208](https://github.com/ClickHouse/ClickHouse/pull/13208) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Function `materialize` (the function for ClickHouse testing) will work for NULL as expected - by transforming it to non-constant column. [#13212](https://github.com/ClickHouse/ClickHouse/pull/13212) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now fast test will wait server with retries. [#13284](https://github.com/ClickHouse/ClickHouse/pull/13284) ([alesapin](https://github.com/alesapin)). +* Applying LDAP authentication test fixes. [#13310](https://github.com/ClickHouse/ClickHouse/pull/13310) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fix timeout error during server restart in the stress test. [#13321](https://github.com/ClickHouse/ClickHouse/pull/13321) ([alesapin](https://github.com/alesapin)). +* - Added testing for RBAC functionality of INSERT privilege in TestFlows. - Expanded tables on which SELECT is being tested. - Added Requirements to match new table engine tests. [#13340](https://github.com/ClickHouse/ClickHouse/pull/13340) ([MyroTk](https://github.com/MyroTk)). +* Remove some of recursive submodules. See [#13378](https://github.com/ClickHouse/ClickHouse/issues/13378). [#13379](https://github.com/ClickHouse/ClickHouse/pull/13379) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Ensure that all the submodules are from proper URLs. Continuation of [#13379](https://github.com/ClickHouse/ClickHouse/issues/13379). This fixes [#13378](https://github.com/ClickHouse/ClickHouse/issues/13378). [#13397](https://github.com/ClickHouse/ClickHouse/pull/13397) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Other +* Avoid re-loading completion from the history file after each query (to avoid history overlaps with other client sessions). [#13086](https://github.com/ClickHouse/ClickHouse/pull/13086) ([Azat Khuzhin](https://github.com/azat)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Bump numpy from 1.18.5 to 1.19.1 in /docs/tools'. [#12655](https://github.com/ClickHouse/ClickHouse/pull/12655) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump urllib3 from 1.25.9 to 1.25.10 in /docs/tools'. [#12703](https://github.com/ClickHouse/ClickHouse/pull/12703) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Revert "Refactor joinGet and implement multi-key lookup."'. [#12708](https://github.com/ClickHouse/ClickHouse/pull/12708) ([alesapin](https://github.com/alesapin)). +* NO CL ENTRY: 'Revert "Abort on std::out_of_range in debug builds"'. [#12752](https://github.com/ClickHouse/ClickHouse/pull/12752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Bump protobuf from 3.12.2 to 3.12.4 in /docs/tools'. [#13102](https://github.com/ClickHouse/ClickHouse/pull/13102) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Merge [#12574](https://github.com/ClickHouse/ClickHouse/issues/12574)'. [#13158](https://github.com/ClickHouse/ClickHouse/pull/13158) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Add QueryTimeMicroseconds, SelectQueryTimeMicroseconds and InsertQuer…"'. [#13303](https://github.com/ClickHouse/ClickHouse/pull/13303) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.7.2.30-stable.md b/docs/changelogs/v20.7.2.30-stable.md new file mode 100644 index 00000000000..a79affdf267 --- /dev/null +++ b/docs/changelogs/v20.7.2.30-stable.md @@ -0,0 +1,41 @@ +### ClickHouse release v20.7.2.30-stable FIXME as compared to v20.7.1.4310-prestable + +#### Performance Improvement +* Backported in [#14188](https://github.com/ClickHouse/ClickHouse/issues/14188): Slightly optimize very short queries with LowCardinality. [#14129](https://github.com/ClickHouse/ClickHouse/pull/14129) ([Anton Popov](https://github.com/CurtizJ)). + +#### Improvement +* Backported in [#13918](https://github.com/ClickHouse/ClickHouse/issues/13918): Fix data race in `lgamma` function. This race was caught only in `tsan`, no side effects a really happened. [#13842](https://github.com/ClickHouse/ClickHouse/pull/13842) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13951](https://github.com/ClickHouse/ClickHouse/issues/13951): Fix wrong error for long queries. It was possible to get syntax error other than `Max query size exceeded` for correct query. [#13928](https://github.com/ClickHouse/ClickHouse/pull/13928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Bug Fix +* Backported in [#14088](https://github.com/ClickHouse/ClickHouse/issues/14088): Removed wrong auth access check when using ClickHouseDictionarySource to query remote tables. [#12756](https://github.com/ClickHouse/ClickHouse/pull/12756) ([sundyli](https://github.com/sundy-li)). +* Backported in [#13559](https://github.com/ClickHouse/ClickHouse/issues/13559): Fix access to redis dictionary after connection was dropped once. It may happen with `cache` and `direct` dictionary layouts. [#13082](https://github.com/ClickHouse/ClickHouse/pull/13082) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#13509](https://github.com/ClickHouse/ClickHouse/issues/13509): Fix parsing row policies from users.xml when names of databases or tables contain dots. This fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779), [#12527](https://github.com/ClickHouse/ClickHouse/issues/12527). [#13199](https://github.com/ClickHouse/ClickHouse/pull/13199) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#13607](https://github.com/ClickHouse/ClickHouse/issues/13607): Fix missing or excessive headers in `TSV/CSVWithNames` formats. This fixes [#12504](https://github.com/ClickHouse/ClickHouse/issues/12504). [#13343](https://github.com/ClickHouse/ClickHouse/pull/13343) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13525](https://github.com/ClickHouse/ClickHouse/issues/13525): Fix possible race in `StorageMemory`. https://clickhouse-test-reports.s3.yandex.net/0/9cac8a7244063d2092ad25d45502611e18d3749c/stress_test_(thread)/stderr.log Have no idea how to write a test. [#13416](https://github.com/ClickHouse/ClickHouse/pull/13416) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13536](https://github.com/ClickHouse/ClickHouse/issues/13536): Fix wrong code in function `netloc`. This fixes [#13335](https://github.com/ClickHouse/ClickHouse/issues/13335). [#13446](https://github.com/ClickHouse/ClickHouse/pull/13446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#14015](https://github.com/ClickHouse/ClickHouse/issues/14015): Fix premature `ON CLUSTER` timeouts for queries that must be executed on a single replica. Fixes [#6704](https://github.com/ClickHouse/ClickHouse/issues/6704), Fixes [#7228](https://github.com/ClickHouse/ClickHouse/issues/7228), Fixes [#13361](https://github.com/ClickHouse/ClickHouse/issues/13361), Fixes [#11884](https://github.com/ClickHouse/ClickHouse/issues/11884). [#13450](https://github.com/ClickHouse/ClickHouse/pull/13450) ([alesapin](https://github.com/alesapin)). +* Backported in [#13551](https://github.com/ClickHouse/ClickHouse/issues/13551): Fix secondary indices corruption in compact parts. [#13538](https://github.com/ClickHouse/ClickHouse/pull/13538) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#13998](https://github.com/ClickHouse/ClickHouse/issues/13998): Fixed the behaviour when sometimes cache-dictionary returned default value instead of present value from source. [#13624](https://github.com/ClickHouse/ClickHouse/pull/13624) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#13664](https://github.com/ClickHouse/ClickHouse/issues/13664): Concurrent `ALTER ... REPLACE/MOVE PARTITION ...` queries might cause deadlock. It's fixed. [#13626](https://github.com/ClickHouse/ClickHouse/pull/13626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#13704](https://github.com/ClickHouse/ClickHouse/issues/13704): Fix typo in error message about `The value of 'number_of_free_entries_in_pool_to_lower_max_size_of_merge' setting`. [#13678](https://github.com/ClickHouse/ClickHouse/pull/13678) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13718](https://github.com/ClickHouse/ClickHouse/issues/13718): Fix crash in JOIN with StorageMerge and `set enable_optimize_predicate_expression=1`. [#13679](https://github.com/ClickHouse/ClickHouse/pull/13679) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#13702](https://github.com/ClickHouse/ClickHouse/issues/13702): Do not optimize any(arrayJoin()) -> arrayJoin() under optimize_move_functions_out_of_any. [#13681](https://github.com/ClickHouse/ClickHouse/pull/13681) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#14107](https://github.com/ClickHouse/ClickHouse/issues/14107): Fix visible data clobbering by progress bar in client in interactive mode. This fixes [#12562](https://github.com/ClickHouse/ClickHouse/issues/12562) and [#13369](https://github.com/ClickHouse/ClickHouse/issues/13369) and [#13584](https://github.com/ClickHouse/ClickHouse/issues/13584) and fixes [#12964](https://github.com/ClickHouse/ClickHouse/issues/12964). [#13691](https://github.com/ClickHouse/ClickHouse/pull/13691) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13734](https://github.com/ClickHouse/ClickHouse/issues/13734): Fix incorrect message in `clickhouse-server.init` while checking user and group. [#13711](https://github.com/ClickHouse/ClickHouse/pull/13711) ([ylchou](https://github.com/ylchou)). +* Backported in [#13773](https://github.com/ClickHouse/ClickHouse/issues/13773): Fix logging Settings.Names/Values when log_queries_min_type > QUERY_START. [#13737](https://github.com/ClickHouse/ClickHouse/pull/13737) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13771](https://github.com/ClickHouse/ClickHouse/issues/13771): Fix race condition between DETACH and background merges. Parts may revive after detach. This is continuation of [#8602](https://github.com/ClickHouse/ClickHouse/issues/8602) that did not fix the issue but introduced a test that started to fail in very rare cases, demonstrating the issue. [#13746](https://github.com/ClickHouse/ClickHouse/pull/13746) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13787](https://github.com/ClickHouse/ClickHouse/issues/13787): Add range check for h3KRing function. This fixes [#13633](https://github.com/ClickHouse/ClickHouse/issues/13633). [#13752](https://github.com/ClickHouse/ClickHouse/pull/13752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13886](https://github.com/ClickHouse/ClickHouse/issues/13886): Fixed `Directory not empty` error when concurrently executing `DROP DATABASE` and `CREATE TABLE`. [#13756](https://github.com/ClickHouse/ClickHouse/pull/13756) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#13800](https://github.com/ClickHouse/ClickHouse/issues/13800): Fix step overflow in range(). [#13790](https://github.com/ClickHouse/ClickHouse/pull/13790) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13813](https://github.com/ClickHouse/ClickHouse/issues/13813): Fix reading from MergeTree table with INDEX of type SET fails when comparing against NULL. This fixes [#13686](https://github.com/ClickHouse/ClickHouse/issues/13686). [#13793](https://github.com/ClickHouse/ClickHouse/pull/13793) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#13834](https://github.com/ClickHouse/ClickHouse/issues/13834): Fix topK/topKWeighted merge (with non-default parameters). [#13817](https://github.com/ClickHouse/ClickHouse/pull/13817) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#13904](https://github.com/ClickHouse/ClickHouse/issues/13904): Fix incorrect sorting for `FixedString` columns. Fixes [#13182](https://github.com/ClickHouse/ClickHouse/issues/13182). [#13887](https://github.com/ClickHouse/ClickHouse/pull/13887) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#13977](https://github.com/ClickHouse/ClickHouse/issues/13977): Fixed potential deadlock when renaming `Distributed` table. [#13922](https://github.com/ClickHouse/ClickHouse/pull/13922) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#14031](https://github.com/ClickHouse/ClickHouse/issues/14031): Fix wrong results in select queries with `DISTINCT` keyword in case `optimize_duplicate_order_by_and_distinct` setting is enabled. [#13925](https://github.com/ClickHouse/ClickHouse/pull/13925) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#14078](https://github.com/ClickHouse/ClickHouse/issues/14078): Fixed wrong mount point in extra info for `Poco::Exception: no space left on device`. [#14050](https://github.com/ClickHouse/ClickHouse/pull/14050) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#14184](https://github.com/ClickHouse/ClickHouse/issues/14184): When waiting for a dictionary update to complete, use the timeout specified by `query_wait_timeout_milliseconds` setting instead of a hard-coded value. [#14105](https://github.com/ClickHouse/ClickHouse/pull/14105) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14169](https://github.com/ClickHouse/ClickHouse/issues/14169): Fix creation of tables with named tuples. This fixes [#13027](https://github.com/ClickHouse/ClickHouse/issues/13027). [#14143](https://github.com/ClickHouse/ClickHouse/pull/14143) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#14245](https://github.com/ClickHouse/ClickHouse/issues/14245): Fixed incorrect sorting order if LowCardinality column. This fixes [#13958](https://github.com/ClickHouse/ClickHouse/issues/13958). [#14223](https://github.com/ClickHouse/ClickHouse/pull/14223) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14250](https://github.com/ClickHouse/ClickHouse/issues/14250): Fix crash in mark inclusion search introduced in https://github.com/ClickHouse/ClickHouse/pull/12277 . [#14225](https://github.com/ClickHouse/ClickHouse/pull/14225) ([Amos Bird](https://github.com/amosbird)). + diff --git a/docs/changelogs/v20.7.3.7-stable.md b/docs/changelogs/v20.7.3.7-stable.md new file mode 100644 index 00000000000..ae888bb5506 --- /dev/null +++ b/docs/changelogs/v20.7.3.7-stable.md @@ -0,0 +1,25 @@ +### ClickHouse release v20.7.3.7-stable FIXME as compared to v20.7.2.30-stable + +#### Improvement +* Backported in [#14361](https://github.com/ClickHouse/ClickHouse/issues/14361): Added Redis requirepass authorization. [#13688](https://github.com/ClickHouse/ClickHouse/pull/13688) ([Ivan Torgashov](https://github.com/it1804)). +* Backported in [#14875](https://github.com/ClickHouse/ClickHouse/issues/14875): Allow using multi-volume storage configuration in storage Distributed. [#14839](https://github.com/ClickHouse/ClickHouse/pull/14839) ([Pavel Kovalenko](https://github.com/Jokser)). + +#### Bug Fix +* Backported in [#14080](https://github.com/ClickHouse/ClickHouse/issues/14080): Fixes /replicas_status endpoint response status code when verbose=1. [#13722](https://github.com/ClickHouse/ClickHouse/pull/13722) ([javi santana](https://github.com/javisantana)). +* Backported in [#14388](https://github.com/ClickHouse/ClickHouse/issues/14388): Fix GRANT ALL statement when executed on a non-global level. [#13987](https://github.com/ClickHouse/ClickHouse/pull/13987) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#14137](https://github.com/ClickHouse/ClickHouse/issues/14137): Fix pointInPolygon with const 2d array as polygon. [#14079](https://github.com/ClickHouse/ClickHouse/pull/14079) ([Alexey Ilyukhov](https://github.com/livace)). +* Backported in [#14292](https://github.com/ClickHouse/ClickHouse/issues/14292): Fix crash when INERT INTO Kafka engine table from an empty result set with a subquery. ... [#14203](https://github.com/ClickHouse/ClickHouse/pull/14203) ([Dongdong Yang](https://github.com/donge)). +* Backported in [#14307](https://github.com/ClickHouse/ClickHouse/issues/14307): Fix segfault in `clickhouse-odbc-bridge` during schema fetch from some external sources. This PR fixes [#13861](https://github.com/ClickHouse/ClickHouse/issues/13861). [#14267](https://github.com/ClickHouse/ClickHouse/pull/14267) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#14342](https://github.com/ClickHouse/ClickHouse/issues/14342): Fix crash during `ALTER` query for table which was created `AS table_function`. Fixes [#14212](https://github.com/ClickHouse/ClickHouse/issues/14212). [#14326](https://github.com/ClickHouse/ClickHouse/pull/14326) ([alesapin](https://github.com/alesapin)). +* Backported in [#14364](https://github.com/ClickHouse/ClickHouse/issues/14364): Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. [#14334](https://github.com/ClickHouse/ClickHouse/pull/14334) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#14507](https://github.com/ClickHouse/ClickHouse/issues/14507): Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. Continuation of [#14334](https://github.com/ClickHouse/ClickHouse/issues/14334). [#14402](https://github.com/ClickHouse/ClickHouse/pull/14402) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#14486](https://github.com/ClickHouse/ClickHouse/issues/14486): Fix bug which leads to wrong merges assignment if table has partitions with a single part. [#14444](https://github.com/ClickHouse/ClickHouse/pull/14444) ([alesapin](https://github.com/alesapin)). +* Backported in [#14482](https://github.com/ClickHouse/ClickHouse/issues/14482): Check for array size overflow in `topK` aggregate function. Without this check the user may send a query with carefully crafter parameters that will lead to server crash. This closes [#14452](https://github.com/ClickHouse/ClickHouse/issues/14452). [#14467](https://github.com/ClickHouse/ClickHouse/pull/14467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#14601](https://github.com/ClickHouse/ClickHouse/issues/14601): Fix rare segfaults in functions with combinator -Resample, which could appear in result of overflow with very large parameters. [#14562](https://github.com/ClickHouse/ClickHouse/pull/14562) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#14728](https://github.com/ClickHouse/ClickHouse/issues/14728): Cleanup data directory after Zookeeper exceptions during CreateQuery for StorageReplicatedMergeTree Engine. [#14563](https://github.com/ClickHouse/ClickHouse/pull/14563) ([Bharat Nallan](https://github.com/bharatnc)). +* Backported in [#14665](https://github.com/ClickHouse/ClickHouse/issues/14665): Fix wrong Decimal multiplication result caused wrong decimal scale of result column. [#14603](https://github.com/ClickHouse/ClickHouse/pull/14603) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#14806](https://github.com/ClickHouse/ClickHouse/issues/14806): Fix bug when `ALTER UPDATE` mutation with Nullable column in assignment expression and constant value (like `UPDATE x = 42`) leads to incorrect value in column or segfault. Fixes [#13634](https://github.com/ClickHouse/ClickHouse/issues/13634), [#14045](https://github.com/ClickHouse/ClickHouse/issues/14045). [#14646](https://github.com/ClickHouse/ClickHouse/pull/14646) ([alesapin](https://github.com/alesapin)). +* Backported in [#14720](https://github.com/ClickHouse/ClickHouse/issues/14720): Fixed missed default database name in metadata of materialized view when executing `ALTER ... MODIFY QUERY`. [#14664](https://github.com/ClickHouse/ClickHouse/pull/14664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#14911](https://github.com/ClickHouse/ClickHouse/issues/14911): Fix SIGSEGV for an attempt to INSERT into StorageFile(fd). [#14887](https://github.com/ClickHouse/ClickHouse/pull/14887) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#14942](https://github.com/ClickHouse/ClickHouse/issues/14942): Fix the issue when some invocations of `extractAllGroups` function may trigger "Memory limit exceeded" error. This fixes [#13383](https://github.com/ClickHouse/ClickHouse/issues/13383). [#14889](https://github.com/ClickHouse/ClickHouse/pull/14889) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.7.4.11-stable.md b/docs/changelogs/v20.7.4.11-stable.md new file mode 100644 index 00000000000..0c8ad1e1639 --- /dev/null +++ b/docs/changelogs/v20.7.4.11-stable.md @@ -0,0 +1,36 @@ +### ClickHouse release v20.7.4.11-stable FIXME as compared to v20.7.3.7-stable + +#### Improvement +* Backported in [#15566](https://github.com/ClickHouse/ClickHouse/issues/15566): Now it's possible to change the type of version column for `VersionedCollapsingMergeTree` with `ALTER` query. [#15442](https://github.com/ClickHouse/ClickHouse/pull/15442) ([alesapin](https://github.com/alesapin)). + +#### Bug Fix +* Backported in [#15019](https://github.com/ClickHouse/ClickHouse/issues/15019): Fixed the incorrect sorting order of `Nullable` column. This fixes [#14344](https://github.com/ClickHouse/ClickHouse/issues/14344). [#14495](https://github.com/ClickHouse/ClickHouse/pull/14495) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14824](https://github.com/ClickHouse/ClickHouse/issues/14824): Fix wrong monotonicity detection for shrunk `Int -> Int` cast of signed types. It might lead to incorrect query result. This bug is unveiled in [#14513](https://github.com/ClickHouse/ClickHouse/issues/14513). [#14783](https://github.com/ClickHouse/ClickHouse/pull/14783) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15149](https://github.com/ClickHouse/ClickHouse/issues/15149): Fix a problem where the server may get stuck on startup while talking to ZooKeeper, if the configuration files have to be fetched from ZK (using the `from_zk` include option). This fixes [#14814](https://github.com/ClickHouse/ClickHouse/issues/14814). [#14843](https://github.com/ClickHouse/ClickHouse/pull/14843) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#15252](https://github.com/ClickHouse/ClickHouse/issues/15252): Fixed segfault in CacheDictionary [#14837](https://github.com/ClickHouse/ClickHouse/issues/14837). [#14879](https://github.com/ClickHouse/ClickHouse/pull/14879) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14988](https://github.com/ClickHouse/ClickHouse/issues/14988): Publish CPU frequencies per logical core in `system.asynchronous_metrics`. This fixes [#14923](https://github.com/ClickHouse/ClickHouse/issues/14923). [#14924](https://github.com/ClickHouse/ClickHouse/pull/14924) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#14970](https://github.com/ClickHouse/ClickHouse/issues/14970): Fix to make predicate push down work when subquery contains finalizeAggregation function. Fixes [#14847](https://github.com/ClickHouse/ClickHouse/issues/14847). [#14937](https://github.com/ClickHouse/ClickHouse/pull/14937) ([filimonov](https://github.com/filimonov)). +* Backported in [#15054](https://github.com/ClickHouse/ClickHouse/issues/15054): Now settings `number_of_free_entries_in_pool_to_execute_mutation` and `number_of_free_entries_in_pool_to_lower_max_size_of_merge` can be equal to `background_pool_size`. [#14975](https://github.com/ClickHouse/ClickHouse/pull/14975) ([alesapin](https://github.com/alesapin)). +* Backported in [#15078](https://github.com/ClickHouse/ClickHouse/issues/15078): Fix crash in RIGHT or FULL JOIN with join_algorith='auto' when memory limit exceeded and we should change HashJoin with MergeJoin. [#15002](https://github.com/ClickHouse/ClickHouse/pull/15002) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15103](https://github.com/ClickHouse/ClickHouse/issues/15103): Fixed `Cannot rename ... errno: 22, strerror: Invalid argument` error on DDL query execution in Atomic database when running clickhouse-server in docker on Mac OS. [#15024](https://github.com/ClickHouse/ClickHouse/pull/15024) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15058](https://github.com/ClickHouse/ClickHouse/issues/15058): If function `bar` was called with specifically crafted arguments, buffer overflow was possible. This closes [#13926](https://github.com/ClickHouse/ClickHouse/issues/13926). [#15028](https://github.com/ClickHouse/ClickHouse/pull/15028) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15051](https://github.com/ClickHouse/ClickHouse/issues/15051): We already use padded comparison between String and FixedString (https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionsComparison.h#L333). This PR applies the same logic to field comparison which corrects the usage of FixedString as primary keys. This fixes [#14908](https://github.com/ClickHouse/ClickHouse/issues/14908). [#15033](https://github.com/ClickHouse/ClickHouse/pull/15033) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15140](https://github.com/ClickHouse/ClickHouse/issues/15140): Fixes `Data compressed with different methods` in `join_algorithm='auto'`. Keep LowCardinality as type for left table join key in `join_algorithm='partial_merge'`. [#15088](https://github.com/ClickHouse/ClickHouse/pull/15088) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15221](https://github.com/ClickHouse/ClickHouse/issues/15221): Fix bug in table engine `Buffer` which doesn't allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). +* Backported in [#15406](https://github.com/ClickHouse/ClickHouse/issues/15406): Fix instance crash when using joinGet with LowCardinality types. This fixes [#15214](https://github.com/ClickHouse/ClickHouse/issues/15214). [#15220](https://github.com/ClickHouse/ClickHouse/pull/15220) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15488](https://github.com/ClickHouse/ClickHouse/issues/15488): Fix 'Unknown identifier' in GROUP BY when query has JOIN over Merge table. [#15242](https://github.com/ClickHouse/ClickHouse/pull/15242) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15280](https://github.com/ClickHouse/ClickHouse/issues/15280): Fix MSan report in QueryLog. Uninitialized memory can be used for the field `memory_usage`. [#15258](https://github.com/ClickHouse/ClickHouse/pull/15258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15415](https://github.com/ClickHouse/ClickHouse/issues/15415): Fix hang of queries with a lot of subqueries to same table of `MySQL` engine. Previously, if there were more than 16 subqueries to same `MySQL` table in query, it hang forever. [#15299](https://github.com/ClickHouse/ClickHouse/pull/15299) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#15337](https://github.com/ClickHouse/ClickHouse/issues/15337): Fix rare race condition on server startup when system.logs are enabled. [#15300](https://github.com/ClickHouse/ClickHouse/pull/15300) ([alesapin](https://github.com/alesapin)). +* Backported in [#15333](https://github.com/ClickHouse/ClickHouse/issues/15333): Fix race condition during MergeTree table rename and background cleanup. [#15304](https://github.com/ClickHouse/ClickHouse/pull/15304) ([alesapin](https://github.com/alesapin)). +* Backported in [#15444](https://github.com/ClickHouse/ClickHouse/issues/15444): Report proper error when the second argument of `boundingRatio` aggregate function has a wrong type. [#15407](https://github.com/ClickHouse/ClickHouse/pull/15407) ([detailyang](https://github.com/detailyang)). +* Backported in [#15506](https://github.com/ClickHouse/ClickHouse/issues/15506): Fix bug with event subscription in DDLWorker which rarely may lead to query hangs in `ON CLUSTER`. Introduced in [#13450](https://github.com/ClickHouse/ClickHouse/issues/13450). [#15477](https://github.com/ClickHouse/ClickHouse/pull/15477) ([alesapin](https://github.com/alesapin)). +* Backported in [#15617](https://github.com/ClickHouse/ClickHouse/issues/15617): Fix `Missing columns` errors when selecting columns which absent in data, but depend on other columns which also absent in data. Fixes [#15530](https://github.com/ClickHouse/ClickHouse/issues/15530). [#15532](https://github.com/ClickHouse/ClickHouse/pull/15532) ([alesapin](https://github.com/alesapin)). +* Backported in [#15559](https://github.com/ClickHouse/ClickHouse/issues/15559): Fix bug when `ILIKE` operator stops being case insensitive if `LIKE` with the same pattern was executed. [#15536](https://github.com/ClickHouse/ClickHouse/pull/15536) ([alesapin](https://github.com/alesapin)). +* Backported in [#15727](https://github.com/ClickHouse/ClickHouse/issues/15727): Mutation might hang waiting for some non-existent part after `MOVE` or `REPLACE PARTITION` or, in rare cases, after `DETACH` or `DROP PARTITION`. It's fixed. [#15537](https://github.com/ClickHouse/ClickHouse/pull/15537) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15653](https://github.com/ClickHouse/ClickHouse/issues/15653): Fix 'Database doesn't exist.' in queries with IN and Distributed table when there's no database on initiator. [#15538](https://github.com/ClickHouse/ClickHouse/pull/15538) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15632](https://github.com/ClickHouse/ClickHouse/issues/15632): Significantly reduce memory usage in AggregatingInOrderTransform/optimize_aggregation_in_order. [#15543](https://github.com/ClickHouse/ClickHouse/pull/15543) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#15584](https://github.com/ClickHouse/ClickHouse/issues/15584): Prevent the possibility of error message `Could not calculate available disk space (statvfs), errno: 4, strerror: Interrupted system call`. This fixes [#15541](https://github.com/ClickHouse/ClickHouse/issues/15541). [#15557](https://github.com/ClickHouse/ClickHouse/pull/15557) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15663](https://github.com/ClickHouse/ClickHouse/issues/15663): Fixed `Element ... is not a constant expression` error when using `JSON*` function result in `VALUES`, `LIMIT` or right side of `IN` operator. [#15589](https://github.com/ClickHouse/ClickHouse/pull/15589) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15716](https://github.com/ClickHouse/ClickHouse/issues/15716): Fix the order of destruction for resources in `ReadFromStorage` step of query plan. It might cause crashes in rare cases. Possibly connected with [#15610](https://github.com/ClickHouse/ClickHouse/issues/15610). [#15645](https://github.com/ClickHouse/ClickHouse/pull/15645) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v20.8.1.4513-prestable.md b/docs/changelogs/v20.8.1.4513-prestable.md new file mode 100644 index 00000000000..0f2385f916c --- /dev/null +++ b/docs/changelogs/v20.8.1.4513-prestable.md @@ -0,0 +1,133 @@ +### ClickHouse release v20.8.1.4513-prestable FIXME as compared to v20.7.1.4310-prestable + +#### Backward Incompatible Change +* Remove support for the `ODBCDriver` input/output format. This was a deprecated format once used for communication with the ClickHouse ODBC driver, now long superseded by the `ODBCDriver2` format. Resolves [#13629](https://github.com/ClickHouse/ClickHouse/issues/13629). [#13847](https://github.com/ClickHouse/ClickHouse/pull/13847) ([hexiaoting](https://github.com/hexiaoting)). +* Extend `parallel_distributed_insert_select` setting, adding an option to run `INSERT` into local table. The setting changes type from `Bool` to `UInt64`, so the values `false` and `true` are no longer supported. If you have these values in server configuration, the server will not start. Please replace them with `0` and `1`, respectively. [#14060](https://github.com/ClickHouse/ClickHouse/pull/14060) ([Azat Khuzhin](https://github.com/azat)). +* Now `OPTIMIZE FINAL` query doesn't recalculate TTL for parts that were added before TTL was created. Use `ALTER TABLE ... MATERIALIZE TTL` once to calculate them, after that `OPTIMIZE FINAL` will evaluate TTL's properly. This behavior never worked for replicated tables. [#14220](https://github.com/ClickHouse/ClickHouse/pull/14220) ([alesapin](https://github.com/alesapin)). + +#### New Feature +* Support `MaterializeMySQL` database engine. Implements [#4006](https://github.com/ClickHouse/ClickHouse/issues/4006). [#10851](https://github.com/ClickHouse/ClickHouse/pull/10851) ([Winter Zhang](https://github.com/zhang2014)). +* Support Kerberos authentication in Kafka, using `krb5` and `cyrus-sasl` libraries. [#12771](https://github.com/ClickHouse/ClickHouse/pull/12771) ([Ilya Golshtein](https://github.com/ilejn)). +* Add types `Int128`, `Int256`, `UInt256` and related functions for them. Extend Decimals with Decimal256 (precision up to 76 digits). New types are under the setting `allow_experimental_bigint_types`. [#13097](https://github.com/ClickHouse/ClickHouse/pull/13097) ([Artem Zuikov](https://github.com/4ertus2)). +* Function `position` now supports optional `start_pos` argument. [#13237](https://github.com/ClickHouse/ClickHouse/pull/13237) ([Vladimir C](https://github.com/vdimir)). +* Add `ALTER SAMPLE BY` statement that allows to change table sample clause. [#13280](https://github.com/ClickHouse/ClickHouse/pull/13280) ([Amos Bird](https://github.com/amosbird)). +* Add new optional section to the main config. [#13425](https://github.com/ClickHouse/ClickHouse/pull/13425) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add function `normalizeQuery` that replaces literals, sequences of literals and complex aliases with placeholders. Add function `normalizedQueryHash` that returns identical 64bit hash values for similar queries. It helps to analyze query log. This closes [#11271](https://github.com/ClickHouse/ClickHouse/issues/11271). [#13816](https://github.com/ClickHouse/ClickHouse/pull/13816) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `quantileExactLow` and `quantileExactHigh` implementations with respective aliases for `medianExactLow` and `medianExactHigh`. [#13818](https://github.com/ClickHouse/ClickHouse/pull/13818) ([Bharat Nallan](https://github.com/bharatnc)). +* Add function `defaultValueOfTypeName` that returns the default value for a given type. [#13877](https://github.com/ClickHouse/ClickHouse/pull/13877) ([hcz](https://github.com/hczhcz)). +* Added `date_trunc` function that truncates a date/time value to a specified date/time part. [#13888](https://github.com/ClickHouse/ClickHouse/pull/13888) ([vladimir-golovchenko](https://github.com/vladimir-golovchenko)). +* Add the ability to specify `Default` compression codec for columns that correspond to settings specified in `config.xml`. Implements: [#9074](https://github.com/ClickHouse/ClickHouse/issues/9074). [#14049](https://github.com/ClickHouse/ClickHouse/pull/14049) ([alesapin](https://github.com/alesapin)). +* Add setting `min_index_granularity_bytes` that protects against accidentally creating a table with very low `index_granularity_bytes` setting. [#14139](https://github.com/ClickHouse/ClickHouse/pull/14139) ([Bharat Nallan](https://github.com/bharatnc)). +* Add `countDigits(x)` function that count number of decimal digits in integer or decimal column. Add `isDecimalOverflow(d, [p])` function that checks if the value in Decimal column is out of its (or specified) precision. [#14151](https://github.com/ClickHouse/ClickHouse/pull/14151) ([Artem Zuikov](https://github.com/4ertus2)). + +#### Performance Improvement +* When performing trivial `INSERT SELECT` queries, automatically set `max_threads` to 1 or `max_insert_threads`, and set `max_block_size` to `min_insert_block_size_rows`. Related to [#5907](https://github.com/ClickHouse/ClickHouse/issues/5907). [#12195](https://github.com/ClickHouse/ClickHouse/pull/12195) ([flynn](https://github.com/ucasfl)). +* Optimize `has()`, `indexOf()` and `countEqual()` functions for `Array(LowCardinality(T))` and constant right arguments. [#12550](https://github.com/ClickHouse/ClickHouse/pull/12550) ([Mike Kot](https://github.com/myrrc)). +* Slightly improve performance of aggregation by UInt8/UInt16 keys. [#13099](https://github.com/ClickHouse/ClickHouse/pull/13099) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fail fast if `max_rows_to_read` limit is exceeded on parts scan. The motivation behind this change is to skip ranges scan for all selected parts if it is clear that `max_rows_to_read` is already exceeded. The change is quite noticeable for queries over big number of parts. [#13677](https://github.com/ClickHouse/ClickHouse/pull/13677) ([Roman Khavronenko](https://github.com/hagen1778)). +* Enable parallel INSERTs for table engines `Null`, `Memory`, `Distributed` and `Buffer`. [#14120](https://github.com/ClickHouse/ClickHouse/pull/14120) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Slightly optimize very short queries with LowCardinality. [#14129](https://github.com/ClickHouse/ClickHouse/pull/14129) ([Anton Popov](https://github.com/CurtizJ)). + +#### Improvement +* Added cache layer for DiskS3 (cache to local disk mark and index files). [#13076](https://github.com/ClickHouse/ClickHouse/pull/13076) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix assert when decimal has too large negative exponent. Fixes [#13188](https://github.com/ClickHouse/ClickHouse/issues/13188). [#13228](https://github.com/ClickHouse/ClickHouse/pull/13228) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add QueryTimeMicroseconds, SelectQueryTimeMicroseconds and InsertQueryTimeMicroseconds to system.events. [#13336](https://github.com/ClickHouse/ClickHouse/pull/13336) ([ianton-ru](https://github.com/ianton-ru)). +* Proper remote host checking in S3 redirects (security-related thing). [#13404](https://github.com/ClickHouse/ClickHouse/pull/13404) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Enable mixed granularity parts by default. [#13449](https://github.com/ClickHouse/ClickHouse/pull/13449) ([alesapin](https://github.com/alesapin)). +* Move parts from DIskLocal to DiskS3 in parallel. [#13459](https://github.com/ClickHouse/ClickHouse/pull/13459) ([Pavel Kovalenko](https://github.com/Jokser)). +* Support compound identifiers for custom settings. [#13496](https://github.com/ClickHouse/ClickHouse/pull/13496) ([Vitaly Baranov](https://github.com/vitlibar)). +* Provide monotonicity for `toDate/toDateTime` functions in more cases. Now the input arguments are saturated more naturally and provides better monotonicity. [#13497](https://github.com/ClickHouse/ClickHouse/pull/13497) ([Amos Bird](https://github.com/amosbird)). +* In previous versions `lcm` function may produce assertion violation in debug build if called with specifically crafted arguments. This fixes [#13368](https://github.com/ClickHouse/ClickHouse/issues/13368). [#13510](https://github.com/ClickHouse/ClickHouse/pull/13510) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add MergeTree Write-Ahead-Log(WAL) dump tool. [#13640](https://github.com/ClickHouse/ClickHouse/pull/13640) ([BohuTANG](https://github.com/BohuTANG)). +* Avoid too slow queries when arrays are manipulated as fields. Throw exception instead. [#13753](https://github.com/ClickHouse/ClickHouse/pull/13753) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* 1. Add [GTID-Based Replication](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html), it works even when replication topology changes, and supported/prefered in MySQL 5.6/5.7/8.0 2. Add BIT/SET filed type supports 3. Fix up varchar type meta length bug. [#13820](https://github.com/ClickHouse/ClickHouse/pull/13820) ([BohuTANG](https://github.com/BohuTANG)). +* Fix data race in `lgamma` function. This race was caught only in `tsan`, no side effects a really happened. [#13842](https://github.com/ClickHouse/ClickHouse/pull/13842) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Function `arrayCompact` will compare NaNs bitwise if the type of array elements is Float32/Float64. In previous versions NaNs were always not equal if the type of array elements is Float32/Float64 and were always equal if the type is more complex, like Nullable(Float64). This closes [#13857](https://github.com/ClickHouse/ClickHouse/issues/13857). [#13868](https://github.com/ClickHouse/ClickHouse/pull/13868) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better error message for null value of TabSeparatedRow format. [#13906](https://github.com/ClickHouse/ClickHouse/pull/13906) ([jiang tao](https://github.com/tomjiang1987)). +* Fix wrong error for long queries. It was possible to get syntax error other than `Max query size exceeded` for correct query. [#13928](https://github.com/ClickHouse/ClickHouse/pull/13928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Corrected an error in AvroConfluent format that caused the Kafka table engine to stop processing messages when an abnormally small, malformed, message was received. [#13941](https://github.com/ClickHouse/ClickHouse/pull/13941) ([Gervasio Varela](https://github.com/gervarela)). +* Increase limit in -Resample combinator to 1M. [#13947](https://github.com/ClickHouse/ClickHouse/pull/13947) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Conditional aggregate functions (for example: `avgIf`, `sumIf`, `maxIf`) should return `NULL` when miss rows and use nullable arguments. [#13964](https://github.com/ClickHouse/ClickHouse/pull/13964) ([Winter Zhang](https://github.com/zhang2014)). +* Slightly better performance of Memory table if it was constructed from a huge number of very small blocks (that's unlikely). Author of the idea: [Mark Papadakis](https://github.com/markpapadakis). Closes [#14043](https://github.com/ClickHouse/ClickHouse/issues/14043). [#14056](https://github.com/ClickHouse/ClickHouse/pull/14056) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `toDeicmal256()` and attempt to fix potential perf regression in Decimal to Float conversion. [#14110](https://github.com/ClickHouse/ClickHouse/pull/14110) ([Artem Zuikov](https://github.com/4ertus2)). +* Now it's possible to `ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'zk://:/path-in-zookeeper'`. It's useful for shipping data to new clusters. [#14155](https://github.com/ClickHouse/ClickHouse/pull/14155) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix +* subquery hash values are not enough to distinguish. [#8333](https://github.com/ClickHouse/ClickHouse/issues/8333). [#8367](https://github.com/ClickHouse/ClickHouse/pull/8367) ([Amos Bird](https://github.com/amosbird)). +* Removed wrong auth access check when using ClickHouseDictionarySource to query remote tables. [#12756](https://github.com/ClickHouse/ClickHouse/pull/12756) ([sundyli](https://github.com/sundy-li)). +* Fix access to redis dictionary after connection was dropped once. It may happen with `cache` and `direct` dictionary layouts. [#13082](https://github.com/ClickHouse/ClickHouse/pull/13082) ([Anton Popov](https://github.com/CurtizJ)). +* Fix parsing row policies from users.xml when names of databases or tables contain dots. This fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779), [#12527](https://github.com/ClickHouse/ClickHouse/issues/12527). [#13199](https://github.com/ClickHouse/ClickHouse/pull/13199) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix missing or excessive headers in `TSV/CSVWithNames` formats. This fixes [#12504](https://github.com/ClickHouse/ClickHouse/issues/12504). [#13343](https://github.com/ClickHouse/ClickHouse/pull/13343) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible race in `StorageMemory`. https://clickhouse-test-reports.s3.yandex.net/0/9cac8a7244063d2092ad25d45502611e18d3749c/stress_test_(thread)/stderr.log Have no idea how to write a test. [#13416](https://github.com/ClickHouse/ClickHouse/pull/13416) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix wrong code in function `netloc`. This fixes [#13335](https://github.com/ClickHouse/ClickHouse/issues/13335). [#13446](https://github.com/ClickHouse/ClickHouse/pull/13446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix premature `ON CLUSTER` timeouts for queries that must be executed on a single replica. Fixes [#6704](https://github.com/ClickHouse/ClickHouse/issues/6704), Fixes [#7228](https://github.com/ClickHouse/ClickHouse/issues/7228), Fixes [#13361](https://github.com/ClickHouse/ClickHouse/issues/13361), Fixes [#11884](https://github.com/ClickHouse/ClickHouse/issues/11884). [#13450](https://github.com/ClickHouse/ClickHouse/pull/13450) ([alesapin](https://github.com/alesapin)). +* Fix secondary indices corruption in compact parts. [#13538](https://github.com/ClickHouse/ClickHouse/pull/13538) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed the behaviour when sometimes cache-dictionary returned default value instead of present value from source. [#13624](https://github.com/ClickHouse/ClickHouse/pull/13624) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Concurrent `ALTER ... REPLACE/MOVE PARTITION ...` queries might cause deadlock. It's fixed. [#13626](https://github.com/ClickHouse/ClickHouse/pull/13626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix typo in error message about `The value of 'number_of_free_entries_in_pool_to_lower_max_size_of_merge' setting`. [#13678](https://github.com/ClickHouse/ClickHouse/pull/13678) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix crash in JOIN with StorageMerge and `set enable_optimize_predicate_expression=1`. [#13679](https://github.com/ClickHouse/ClickHouse/pull/13679) ([Artem Zuikov](https://github.com/4ertus2)). +* Do not optimize any(arrayJoin()) -> arrayJoin() under optimize_move_functions_out_of_any. [#13681](https://github.com/ClickHouse/ClickHouse/pull/13681) ([Azat Khuzhin](https://github.com/azat)). +* Fix visible data clobbering by progress bar in client in interactive mode. This fixes [#12562](https://github.com/ClickHouse/ClickHouse/issues/12562) and [#13369](https://github.com/ClickHouse/ClickHouse/issues/13369) and [#13584](https://github.com/ClickHouse/ClickHouse/issues/13584) and fixes [#12964](https://github.com/ClickHouse/ClickHouse/issues/12964). [#13691](https://github.com/ClickHouse/ClickHouse/pull/13691) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix incorrect message in `clickhouse-server.init` while checking user and group. [#13711](https://github.com/ClickHouse/ClickHouse/pull/13711) ([ylchou](https://github.com/ylchou)). +* Fixes /replicas_status endpoint response status code when verbose=1. [#13722](https://github.com/ClickHouse/ClickHouse/pull/13722) ([javi santana](https://github.com/javisantana)). +* Fix logging Settings.Names/Values when log_queries_min_type > QUERY_START. [#13737](https://github.com/ClickHouse/ClickHouse/pull/13737) ([Azat Khuzhin](https://github.com/azat)). +* Fix race condition between DETACH and background merges. Parts may revive after detach. This is continuation of [#8602](https://github.com/ClickHouse/ClickHouse/issues/8602) that did not fix the issue but introduced a test that started to fail in very rare cases, demonstrating the issue. [#13746](https://github.com/ClickHouse/ClickHouse/pull/13746) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add range check for h3KRing function. This fixes [#13633](https://github.com/ClickHouse/ClickHouse/issues/13633). [#13752](https://github.com/ClickHouse/ClickHouse/pull/13752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed `Directory not empty` error when concurrently executing `DROP DATABASE` and `CREATE TABLE`. [#13756](https://github.com/ClickHouse/ClickHouse/pull/13756) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix step overflow in range(). [#13790](https://github.com/ClickHouse/ClickHouse/pull/13790) ([Azat Khuzhin](https://github.com/azat)). +* Fix reading from MergeTree table with INDEX of type SET fails when comparing against NULL. This fixes [#13686](https://github.com/ClickHouse/ClickHouse/issues/13686). [#13793](https://github.com/ClickHouse/ClickHouse/pull/13793) ([Amos Bird](https://github.com/amosbird)). +* Fix topK/topKWeighted merge (with non-default parameters). [#13817](https://github.com/ClickHouse/ClickHouse/pull/13817) ([Azat Khuzhin](https://github.com/azat)). +* Fix incorrect sorting for `FixedString` columns. Fixes [#13182](https://github.com/ClickHouse/ClickHouse/issues/13182). [#13887](https://github.com/ClickHouse/ClickHouse/pull/13887) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed potential deadlock when renaming `Distributed` table. [#13922](https://github.com/ClickHouse/ClickHouse/pull/13922) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix wrong results in select queries with `DISTINCT` keyword in case `optimize_duplicate_order_by_and_distinct` setting is enabled. [#13925](https://github.com/ClickHouse/ClickHouse/pull/13925) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix parser to reject create table as table function with engine. [#13940](https://github.com/ClickHouse/ClickHouse/pull/13940) ([hcz](https://github.com/hczhcz)). +* Fixed wrong mount point in extra info for `Poco::Exception: no space left on device`. [#14050](https://github.com/ClickHouse/ClickHouse/pull/14050) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix pointInPolygon with const 2d array as polygon. [#14079](https://github.com/ClickHouse/ClickHouse/pull/14079) ([Alexey Ilyukhov](https://github.com/livace)). +* Fix DistributedFilesToInsert metric (zeroed when it should not). [#14095](https://github.com/ClickHouse/ClickHouse/pull/14095) ([Azat Khuzhin](https://github.com/azat)). +* When waiting for a dictionary update to complete, use the timeout specified by `query_wait_timeout_milliseconds` setting instead of a hard-coded value. [#14105](https://github.com/ClickHouse/ClickHouse/pull/14105) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix formatting of minimal negative decimal numbers. This fixes [#14111](https://github.com/ClickHouse/ClickHouse/issues/14111). [#14119](https://github.com/ClickHouse/ClickHouse/pull/14119) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix creation of tables with named tuples. This fixes [#13027](https://github.com/ClickHouse/ClickHouse/issues/13027). [#14143](https://github.com/ClickHouse/ClickHouse/pull/14143) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix crash when INERT INTO Kafka engine table from an empty result set with a subquery. ... [#14203](https://github.com/ClickHouse/ClickHouse/pull/14203) ([Dongdong Yang](https://github.com/donge)). +* Fixed incorrect sorting order if LowCardinality column. This fixes [#13958](https://github.com/ClickHouse/ClickHouse/issues/13958). [#14223](https://github.com/ClickHouse/ClickHouse/pull/14223) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix crash in mark inclusion search introduced in https://github.com/ClickHouse/ClickHouse/pull/12277 . [#14225](https://github.com/ClickHouse/ClickHouse/pull/14225) ([Amos Bird](https://github.com/amosbird)). +* fixes [#14231](https://github.com/ClickHouse/ClickHouse/issues/14231) fix wrong lexer in MaterializeMySQL database engine dump stage. [#14232](https://github.com/ClickHouse/ClickHouse/pull/14232) ([Winter Zhang](https://github.com/zhang2014)). +* Fix handling of empty transactions in `MaterializeMySQL` database engine. This fixes [#14235](https://github.com/ClickHouse/ClickHouse/issues/14235). [#14253](https://github.com/ClickHouse/ClickHouse/pull/14253) ([BohuTANG](https://github.com/BohuTANG)). +* Fix segfault in `clickhouse-odbc-bridge` during schema fetch from some external sources. This PR fixes [#13861](https://github.com/ClickHouse/ClickHouse/issues/13861). [#14267](https://github.com/ClickHouse/ClickHouse/pull/14267) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Build/Testing/Packaging Improvement +* Move Dockerfiles from integration tests to `docker/test` directory. docker_compose files are available in `runner` docker container. Docker images are built in CI and not in integration tests. [#13448](https://github.com/ClickHouse/ClickHouse/pull/13448) ([Ilya Yatsishin](https://github.com/qoega)). +* Skip PR's from robot-clickhouse. [#13489](https://github.com/ClickHouse/ClickHouse/pull/13489) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix typos in code with codespell. [#13511](https://github.com/ClickHouse/ClickHouse/pull/13511) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable check for typos in code with `codespell`. [#13513](https://github.com/ClickHouse/ClickHouse/pull/13513) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Ensure that there is no copy-pasted GPL code. [#13514](https://github.com/ClickHouse/ClickHouse/pull/13514) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to run `clickhouse` binary without configuration. [#13515](https://github.com/ClickHouse/ClickHouse/pull/13515) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added `clickhouse install` script, that is useful if you only have a single binary. [#13528](https://github.com/ClickHouse/ClickHouse/pull/13528) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix the remaining shellcheck notices. A preparation to enable Shellcheck. [#13529](https://github.com/ClickHouse/ClickHouse/pull/13529) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable Shellcheck in CI as a linter of .sh tests. This closes [#13168](https://github.com/ClickHouse/ClickHouse/issues/13168). [#13530](https://github.com/ClickHouse/ClickHouse/pull/13530) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make sure [#10977](https://github.com/ClickHouse/ClickHouse/issues/10977) is invalid. [#13539](https://github.com/ClickHouse/ClickHouse/pull/13539) ([Amos Bird](https://github.com/amosbird)). +* Increasing health-check timeouts for ClickHouse nodes and adding support to dump docker-compose logs if unhealthy containers found. [#13612](https://github.com/ClickHouse/ClickHouse/pull/13612) ([vzakaznikov](https://github.com/vzakaznikov)). +* Build ClickHouse with the most fresh tzdata from package repository. [#13623](https://github.com/ClickHouse/ClickHouse/pull/13623) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Removed `-DENABLE_CURL_CLIENT` for `contrib/aws`. [#13628](https://github.com/ClickHouse/ClickHouse/pull/13628) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Expose version of embedded tzdata via TZDATA_VERSION in system.build_options. [#13648](https://github.com/ClickHouse/ClickHouse/pull/13648) ([filimonov](https://github.com/filimonov)). +* Updating LDAP user authentication suite to check that it works with RBAC. [#13656](https://github.com/ClickHouse/ClickHouse/pull/13656) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add a CMake option to fail configuration instead of auto-reconfiguration, enabled by default. [#13687](https://github.com/ClickHouse/ClickHouse/pull/13687) ([Konstantin](https://github.com/podshumok)). +* Fix link error in shared build. [#13700](https://github.com/ClickHouse/ClickHouse/pull/13700) ([Amos Bird](https://github.com/amosbird)). +* FIx cassandra build on Mac OS. [#13708](https://github.com/ClickHouse/ClickHouse/pull/13708) ([Ilya Yatsishin](https://github.com/qoega)). +* Added docker image for style check. Added style check that all docker and docker compose files are located in docker directory. [#13724](https://github.com/ClickHouse/ClickHouse/pull/13724) ([Ilya Yatsishin](https://github.com/qoega)). +* ZooKeeper cannot work reliably in unit tests in CI infrastructure. Using unit tests for ZooKeeper interaction with real ZooKeeper is bad idea from the start (unit tests are not supposed to verify complex distributed systems). We already using integration tests for this purpose and they are better suited. [#13745](https://github.com/ClickHouse/ClickHouse/pull/13745) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Testflows LDAP module: adding missing certificates and dhparam.pem for openldap4. [#13780](https://github.com/ClickHouse/ClickHouse/pull/13780) ([vzakaznikov](https://github.com/vzakaznikov)). +* Enabled text-log in stress test to find more bugs. [#13855](https://github.com/ClickHouse/ClickHouse/pull/13855) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* * Adding retry logic when bringing up docker-compose cluster * Increasing COMPOSE_HTTP_TIMEOUT. [#14112](https://github.com/ClickHouse/ClickHouse/pull/14112) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add the ability to write js-style comments in skip_list.json. [#14159](https://github.com/ClickHouse/ClickHouse/pull/14159) ([alesapin](https://github.com/alesapin)). +* Switch tests docker images to use test-base parent. [#14167](https://github.com/ClickHouse/ClickHouse/pull/14167) ([Ilya Yatsishin](https://github.com/qoega)). +* Actually there are no symlinks there, so `-type f` is enough ``` ~/workspace/ClickHouse/contrib/cctz/testdata/zoneinfo$ find . -type l -ls | wc -l 0 ``` Closes [#14209](https://github.com/ClickHouse/ClickHouse/issues/14209). [#14215](https://github.com/ClickHouse/ClickHouse/pull/14215) ([filimonov](https://github.com/filimonov)). + +#### Other +* Fix readline so it dumps history to file now. [#13600](https://github.com/ClickHouse/ClickHouse/pull/13600) ([Amos Bird](https://github.com/amosbird)). +* Create `system` database with `Atomic` engine by default. [#13680](https://github.com/ClickHouse/ClickHouse/pull/13680) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Bump pymdown-extensions from 7.1 to 8.0 in /docs/tools'. [#13645](https://github.com/ClickHouse/ClickHouse/pull/13645) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump protobuf from 3.12.4 to 3.13.0 in /docs/tools'. [#13824](https://github.com/ClickHouse/ClickHouse/pull/13824) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). + diff --git a/docs/changelogs/v20.8.10.13-lts.md b/docs/changelogs/v20.8.10.13-lts.md new file mode 100644 index 00000000000..b6c69e115a9 --- /dev/null +++ b/docs/changelogs/v20.8.10.13-lts.md @@ -0,0 +1,6 @@ +### ClickHouse release v20.8.10.13-lts FIXME as compared to v20.8.9.6-lts + +#### Bug Fix +* Backported in [#17967](https://github.com/ClickHouse/ClickHouse/issues/17967): Fix [#15235](https://github.com/ClickHouse/ClickHouse/issues/15235). When clickhouse-copier handle non-partitioned table, throws segfault error. [#17248](https://github.com/ClickHouse/ClickHouse/pull/17248) ([Qi Chen](https://github.com/kaka11chen)). +* Backported in [#18155](https://github.com/ClickHouse/ClickHouse/issues/18155): fix incorrect initialize `max_compress_block_size` of MergeTreeWriterSettings with `min_compress_block_size`. [#17833](https://github.com/ClickHouse/ClickHouse/pull/17833) ([flynn](https://github.com/ucasfl)). + diff --git a/docs/changelogs/v20.8.11.17-lts.md b/docs/changelogs/v20.8.11.17-lts.md new file mode 100644 index 00000000000..8416df896d6 --- /dev/null +++ b/docs/changelogs/v20.8.11.17-lts.md @@ -0,0 +1,5 @@ +### ClickHouse release v20.8.11.17-lts FIXME as compared to v20.8.10.13-lts + +#### Bug Fix +* Backported in [#18442](https://github.com/ClickHouse/ClickHouse/issues/18442): Fix possible crashes in aggregate functions with combinator `Distinct`, while using two-level aggregation. Fixes [#17682](https://github.com/ClickHouse/ClickHouse/issues/17682). [#18365](https://github.com/ClickHouse/ClickHouse/pull/18365) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v20.8.12.2-lts.md b/docs/changelogs/v20.8.12.2-lts.md new file mode 100644 index 00000000000..855ef016031 --- /dev/null +++ b/docs/changelogs/v20.8.12.2-lts.md @@ -0,0 +1,5 @@ +### ClickHouse release v20.8.12.2-lts FIXME as compared to v20.8.11.17-lts + +#### Bug Fix +* Backported in [#18514](https://github.com/ClickHouse/ClickHouse/issues/18514): Restrict merges from wide to compact parts. In case of vertical merge it led to broken result part. [#18381](https://github.com/ClickHouse/ClickHouse/pull/18381) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v20.8.13.15-lts.md b/docs/changelogs/v20.8.13.15-lts.md new file mode 100644 index 00000000000..7635f641d92 --- /dev/null +++ b/docs/changelogs/v20.8.13.15-lts.md @@ -0,0 +1,42 @@ +### ClickHouse release v20.8.13.15-lts FIXME as compared to v20.8.12.2-lts + +#### Backward Incompatible Change +* Backported in [#20418](https://github.com/ClickHouse/ClickHouse/issues/20418): Now it's not allowed to create MergeTree tables in old syntax with table TTL because it's just ignored. Attach of old tables is still possible. [#20282](https://github.com/ClickHouse/ClickHouse/pull/20282) ([alesapin](https://github.com/alesapin)). + +#### Performance Improvement +* Backported in [#20926](https://github.com/ClickHouse/ClickHouse/issues/20926): Fix performance of reading from `Merge` tables over huge number of `MergeTree` tables. Fixes [#7748](https://github.com/ClickHouse/ClickHouse/issues/7748). [#16988](https://github.com/ClickHouse/ClickHouse/pull/16988) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix +* Backported in [#20953](https://github.com/ClickHouse/ClickHouse/issues/20953): Return number of affected rows for INSERT queries via MySQL protocol. Previously ClickHouse used to always return 0, it's fixed. Fixes [#16605](https://github.com/ClickHouse/ClickHouse/issues/16605). [#16715](https://github.com/ClickHouse/ClickHouse/pull/16715) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#20951](https://github.com/ClickHouse/ClickHouse/issues/20951): TODO. [#16866](https://github.com/ClickHouse/ClickHouse/pull/16866) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#20918](https://github.com/ClickHouse/ClickHouse/issues/20918): Fix indeterministic functions with predicate optimizer. This fixes [#17244](https://github.com/ClickHouse/ClickHouse/issues/17244). [#17273](https://github.com/ClickHouse/ClickHouse/pull/17273) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#20924](https://github.com/ClickHouse/ClickHouse/issues/20924): Fix alter query hang when the corresponding mutation was killed on the different replica. Fixes [#16953](https://github.com/ClickHouse/ClickHouse/issues/16953). [#17499](https://github.com/ClickHouse/ClickHouse/pull/17499) ([alesapin](https://github.com/alesapin)). +* Backported in [#20915](https://github.com/ClickHouse/ClickHouse/issues/20915): Fix empty `system.stack_trace` table when server is running in daemon mode. [#17630](https://github.com/ClickHouse/ClickHouse/pull/17630) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#20922](https://github.com/ClickHouse/ClickHouse/issues/20922): Fixed segfault when there is not enough space when inserting into `Distributed` table. [#17737](https://github.com/ClickHouse/ClickHouse/pull/17737) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#20920](https://github.com/ClickHouse/ClickHouse/issues/20920): Fix possible segfault in `topK` aggregate function. This closes [#17404](https://github.com/ClickHouse/ClickHouse/issues/17404). [#17845](https://github.com/ClickHouse/ClickHouse/pull/17845) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#20919](https://github.com/ClickHouse/ClickHouse/issues/20919): Fixed `std::out_of_range: basic_string` in S3 URL parsing. [#18059](https://github.com/ClickHouse/ClickHouse/pull/18059) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#20917](https://github.com/ClickHouse/ClickHouse/issues/20917): - Fixed issue when `clickhouse-odbc-bridge` process is unreachable by server on machines with dual IPv4/IPv6 stack; - Fixed issue when ODBC dictionary updates are performed using malformed queries and/or cause crashes; Possibly closes [#14489](https://github.com/ClickHouse/ClickHouse/issues/14489). [#18278](https://github.com/ClickHouse/ClickHouse/pull/18278) ([Denis Glazachev](https://github.com/traceon)). +* Backported in [#20914](https://github.com/ClickHouse/ClickHouse/issues/20914): Fix filling table `system.settings_profile_elements`. This PR fixes [#18231](https://github.com/ClickHouse/ClickHouse/issues/18231). [#18379](https://github.com/ClickHouse/ClickHouse/pull/18379) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#20913](https://github.com/ClickHouse/ClickHouse/issues/20913): Disable write with AIO during merges because it can lead to extremely rare data corruption of primary key columns during merge. [#18481](https://github.com/ClickHouse/ClickHouse/pull/18481) ([alesapin](https://github.com/alesapin)). +* Backported in [#20912](https://github.com/ClickHouse/ClickHouse/issues/20912): Fix bug which may lead to `ALTER` queries hung after corresponding mutation kill. Found by thread fuzzer. [#18518](https://github.com/ClickHouse/ClickHouse/pull/18518) ([alesapin](https://github.com/alesapin)). +* Backported in [#20910](https://github.com/ClickHouse/ClickHouse/issues/20910): Fix previous bug when date overflow with different values. Strict Date value limit to "2106-02-07", cast date > "2106-02-07" to value 0. [#18565](https://github.com/ClickHouse/ClickHouse/pull/18565) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#20907](https://github.com/ClickHouse/ClickHouse/issues/20907): Fix Logger with unmatched arg size. [#18717](https://github.com/ClickHouse/ClickHouse/pull/18717) ([sundyli](https://github.com/sundy-li)). +* Backported in [#20905](https://github.com/ClickHouse/ClickHouse/issues/20905): Fix bug when mutation with some escaped text (like `ALTER ... UPDATE e = CAST('foo', 'Enum8(\'foo\' = 1')` serialized incorrectly. Fixes [#18878](https://github.com/ClickHouse/ClickHouse/issues/18878). [#18944](https://github.com/ClickHouse/ClickHouse/pull/18944) ([alesapin](https://github.com/alesapin)). +* Backported in [#20904](https://github.com/ClickHouse/ClickHouse/issues/20904): Make sure `groupUniqArray` returns correct type for argument of Enum type. This closes [#17875](https://github.com/ClickHouse/ClickHouse/issues/17875). [#19019](https://github.com/ClickHouse/ClickHouse/pull/19019) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#20901](https://github.com/ClickHouse/ClickHouse/issues/20901): Restrict `MODIFY TTL` queries for `MergeTree` tables created in old syntax. Previously the query succeeded, but actually it had no effect. [#19064](https://github.com/ClickHouse/ClickHouse/pull/19064) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#20900](https://github.com/ClickHouse/ClickHouse/issues/20900): Fix infinite reading from file in `ORC` format (was introduced in [#10580](https://github.com/ClickHouse/ClickHouse/issues/10580)). Fixes [#19095](https://github.com/ClickHouse/ClickHouse/issues/19095). [#19134](https://github.com/ClickHouse/ClickHouse/pull/19134) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#20897](https://github.com/ClickHouse/ClickHouse/issues/20897): - Fix default value in join types with non-zero default (e.g. some Enums). Closes [#18197](https://github.com/ClickHouse/ClickHouse/issues/18197). [#19360](https://github.com/ClickHouse/ClickHouse/pull/19360) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#20898](https://github.com/ClickHouse/ClickHouse/issues/20898): Fix possible buffer overflow in Uber H3 library. See https://github.com/uber/h3/issues/392. This closes [#19219](https://github.com/ClickHouse/ClickHouse/issues/19219). [#19383](https://github.com/ClickHouse/ClickHouse/pull/19383) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#20894](https://github.com/ClickHouse/ClickHouse/issues/20894): Fixed very rare bug that might cause mutation to hang after `DROP/DETACH/REPLACE/MOVE PARTITION`. It was partially fixed by [#15537](https://github.com/ClickHouse/ClickHouse/issues/15537) for the most cases. [#19443](https://github.com/ClickHouse/ClickHouse/pull/19443) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#20896](https://github.com/ClickHouse/ClickHouse/issues/20896): Buffer overflow (on memory read) was possible if `addMonth` function was called with specifically crafted arguments. This fixes [#19441](https://github.com/ClickHouse/ClickHouse/issues/19441). This fixes [#19413](https://github.com/ClickHouse/ClickHouse/issues/19413). [#19472](https://github.com/ClickHouse/ClickHouse/pull/19472) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#20893](https://github.com/ClickHouse/ClickHouse/issues/20893): Query CREATE DICTIONARY id expression fix. [#19571](https://github.com/ClickHouse/ClickHouse/pull/19571) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#20890](https://github.com/ClickHouse/ClickHouse/issues/20890): Fix wrong result of function `neighbor` for `LowCardinality` argument. Fixes [#10333](https://github.com/ClickHouse/ClickHouse/issues/10333). [#19617](https://github.com/ClickHouse/ClickHouse/pull/19617) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#20525](https://github.com/ClickHouse/ClickHouse/issues/20525): Background thread which executes `ON CLUSTER` queries might hang waiting for dropped replicated table to do something. It's fixed. [#19684](https://github.com/ClickHouse/ClickHouse/pull/19684) ([yiguolei](https://github.com/yiguolei)). +* Backported in [#20891](https://github.com/ClickHouse/ClickHouse/issues/20891): Fix a segmentation fault in `bitmapAndnot` function. Fixes [#19668](https://github.com/ClickHouse/ClickHouse/issues/19668). [#19713](https://github.com/ClickHouse/ClickHouse/pull/19713) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#20889](https://github.com/ClickHouse/ClickHouse/issues/20889): In previous versions, unusual arguments for function arrayEnumerateUniq may cause crash or infinite loop. This closes [#19787](https://github.com/ClickHouse/ClickHouse/issues/19787). [#19788](https://github.com/ClickHouse/ClickHouse/pull/19788) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#20521](https://github.com/ClickHouse/ClickHouse/issues/20521): The `MongoDB` table engine now establishes connection only when it's going to read data. `ATTACH TABLE` won't try to connect anymore. [#20110](https://github.com/ClickHouse/ClickHouse/pull/20110) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#20533](https://github.com/ClickHouse/ClickHouse/issues/20533): Fixed the behavior when in case of broken JSON we tried to read the whole file into memory which leads to exception from the allocator. Fixes [#19719](https://github.com/ClickHouse/ClickHouse/issues/19719). [#20286](https://github.com/ClickHouse/ClickHouse/pull/20286) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#20414](https://github.com/ClickHouse/ClickHouse/issues/20414): Restrict to `DROP` or `RENAME` version column of `*CollapsingMergeTree` and `ReplacingMergeTree` table engines. [#20300](https://github.com/ClickHouse/ClickHouse/pull/20300) ([alesapin](https://github.com/alesapin)). +* Backported in [#20522](https://github.com/ClickHouse/ClickHouse/issues/20522): Avoid invalid dereference in RANGE_HASHED() dictionary. [#20345](https://github.com/ClickHouse/ClickHouse/pull/20345) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#20591](https://github.com/ClickHouse/ClickHouse/issues/20591): Fixed inconsistent behavior of dictionary in case of queries where we look for absent keys in dictionary. [#20578](https://github.com/ClickHouse/ClickHouse/pull/20578) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v20.8.14.4-lts.md b/docs/changelogs/v20.8.14.4-lts.md new file mode 100644 index 00000000000..0ac5d3fa713 --- /dev/null +++ b/docs/changelogs/v20.8.14.4-lts.md @@ -0,0 +1,19 @@ +### ClickHouse release v20.8.14.4-lts FIXME as compared to v20.8.13.15-lts + +#### Improvement +* Backported in [#21235](https://github.com/ClickHouse/ClickHouse/issues/21235): The value of MYSQL_OPT_RECONNECT option can now be controlled by "opt_reconnect" parameter in the config section of mysql replica. [#19998](https://github.com/ClickHouse/ClickHouse/pull/19998) ([Alexander Kazakov](https://github.com/Akazz)). +* Backported in [#21203](https://github.com/ClickHouse/ClickHouse/issues/21203): When loading config for mysql source ClickHouse will now randomize the list of replicas with the same priority to ensure the round-robin logics of picking mysql endpoint. This closes [#20629](https://github.com/ClickHouse/ClickHouse/issues/20629). [#20632](https://github.com/ClickHouse/ClickHouse/pull/20632) ([Alexander Kazakov](https://github.com/Akazz)). +* Backported in [#21336](https://github.com/ClickHouse/ClickHouse/issues/21336): MySQL dictionary source will now retry unexpected connection failures (Lost connection to MySQL server during query) which sometimes happen on SSL/TLS connections. [#21237](https://github.com/ClickHouse/ClickHouse/pull/21237) ([Alexander Kazakov](https://github.com/Akazz)). + +#### Bug Fix +* Backported in [#20950](https://github.com/ClickHouse/ClickHouse/issues/20950): Fix set index invalidation when there are const columns in the subquery. This fixes [#17246](https://github.com/ClickHouse/ClickHouse/issues/17246) . [#17249](https://github.com/ClickHouse/ClickHouse/pull/17249) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#20948](https://github.com/ClickHouse/ClickHouse/issues/20948): Fix crash while reading from `JOIN` table with `LowCardinality` types. Fixes [#17228](https://github.com/ClickHouse/ClickHouse/issues/17228). [#17397](https://github.com/ClickHouse/ClickHouse/pull/17397) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#20903](https://github.com/ClickHouse/ClickHouse/issues/20903): Fix inserting of `LowCardinality` column to table with `TinyLog` engine. Fixes [#18629](https://github.com/ClickHouse/ClickHouse/issues/18629). [#19010](https://github.com/ClickHouse/ClickHouse/pull/19010) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#20899](https://github.com/ClickHouse/ClickHouse/issues/20899): Fix bug when concurrent `ALTER` and `DROP` queries may hang while processing ReplicatedMergeTree table. [#19237](https://github.com/ClickHouse/ClickHouse/pull/19237) ([alesapin](https://github.com/alesapin)). +* Backported in [#20892](https://github.com/ClickHouse/ClickHouse/issues/20892): Fix use-after-free of the CompressedWriteBuffer in Connection after disconnect. [#19599](https://github.com/ClickHouse/ClickHouse/pull/19599) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#20888](https://github.com/ClickHouse/ClickHouse/issues/20888): Fix rare bug when some replicated operations (like mutation) cannot process some parts after data corruption. Fixes [#19593](https://github.com/ClickHouse/ClickHouse/issues/19593). [#19702](https://github.com/ClickHouse/ClickHouse/pull/19702) ([alesapin](https://github.com/alesapin)). +* Backported in [#21049](https://github.com/ClickHouse/ClickHouse/issues/21049): Fix usage of `-Distinct` combinator with `-State` combinator in aggregate functions. [#20866](https://github.com/ClickHouse/ClickHouse/pull/20866) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#21130](https://github.com/ClickHouse/ClickHouse/issues/21130): Fixed behaviour, when `ALTER MODIFY COLUMN` created mutation, that will knowingly fail. [#21007](https://github.com/ClickHouse/ClickHouse/pull/21007) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#21159](https://github.com/ClickHouse/ClickHouse/issues/21159): Fix `input_format_null_as_default` take effective when types are nullable. This fixes [#21116](https://github.com/ClickHouse/ClickHouse/issues/21116) . [#21121](https://github.com/ClickHouse/ClickHouse/pull/21121) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#21378](https://github.com/ClickHouse/ClickHouse/issues/21378): Fix error `Bad cast from type ... to DB::ColumnLowCardinality` while inserting into table with `LowCardinality` column from `Values` format. Fixes [#21140](https://github.com/ClickHouse/ClickHouse/issues/21140). [#21357](https://github.com/ClickHouse/ClickHouse/pull/21357) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v20.8.15.11-lts.md b/docs/changelogs/v20.8.15.11-lts.md new file mode 100644 index 00000000000..dfdb0cf1ad2 --- /dev/null +++ b/docs/changelogs/v20.8.15.11-lts.md @@ -0,0 +1,17 @@ +### ClickHouse release v20.8.15.11-lts FIXME as compared to v20.8.14.4-lts + +#### Bug Fix +* Backported in [#21627](https://github.com/ClickHouse/ClickHouse/issues/21627): Fix max_distributed_connections (affects `prefer_localhost_replica=1` and `max_threads!=max_distributed_connections`). [#17848](https://github.com/ClickHouse/ClickHouse/pull/17848) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21626](https://github.com/ClickHouse/ClickHouse/issues/21626): Join tries to materialize const columns, but our code waits for them in other places. [#18982](https://github.com/ClickHouse/ClickHouse/pull/18982) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#21409](https://github.com/ClickHouse/ClickHouse/issues/21409): Now mutations allowed only for table engines that support them (MergeTree family, Memory, MaterializedView). Other engines will report a more clear error. Fixes [#21168](https://github.com/ClickHouse/ClickHouse/issues/21168). [#21183](https://github.com/ClickHouse/ClickHouse/pull/21183) ([alesapin](https://github.com/alesapin)). +* Backported in [#22190](https://github.com/ClickHouse/ClickHouse/issues/22190): Fixed race on SSL object inside SecureSocket in Poco. [#21456](https://github.com/ClickHouse/ClickHouse/pull/21456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#21547](https://github.com/ClickHouse/ClickHouse/issues/21547): Fix a deadlock in `ALTER DELETE` mutations for non replicated MergeTree table engines when the predicate contains the table itself. Fixes [#20558](https://github.com/ClickHouse/ClickHouse/issues/20558). [#21477](https://github.com/ClickHouse/ClickHouse/pull/21477) ([alesapin](https://github.com/alesapin)). +* Backported in [#22284](https://github.com/ClickHouse/ClickHouse/issues/22284): Fix bug for ReplicatedMerge table engines when `ALTER MODIFY COLUMN` query doesn't change the type of decimal column if its size (32 bit or 64 bit) doesn't change. [#21728](https://github.com/ClickHouse/ClickHouse/pull/21728) ([alesapin](https://github.com/alesapin)). +* Backported in [#22282](https://github.com/ClickHouse/ClickHouse/issues/22282): Docker entrypoint: avoid chown of `.` in case when `LOG_PATH` is empty. Closes [#22100](https://github.com/ClickHouse/ClickHouse/issues/22100). [#22102](https://github.com/ClickHouse/ClickHouse/pull/22102) ([filimonov](https://github.com/filimonov)). +* Backported in [#22279](https://github.com/ClickHouse/ClickHouse/issues/22279): Fix waiting for `OPTIMIZE` and `ALTER` queries for `ReplicatedMergeTree` table engines. Now the query will not hang when the table was detached or restarted. [#22118](https://github.com/ClickHouse/ClickHouse/pull/22118) ([alesapin](https://github.com/alesapin)). +* Backported in [#22370](https://github.com/ClickHouse/ClickHouse/issues/22370): Now clickhouse will not throw `LOGICAL_ERROR` exception when we try to mutate the already covered part. Fixes [#22013](https://github.com/ClickHouse/ClickHouse/issues/22013). [#22291](https://github.com/ClickHouse/ClickHouse/pull/22291) ([alesapin](https://github.com/alesapin)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Backport [#17848](https://github.com/ClickHouse/ClickHouse/issues/17848) to 20.8: Fix max_distributed_connections"'. [#21940](https://github.com/ClickHouse/ClickHouse/pull/21940) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v20.8.16.20-lts.md b/docs/changelogs/v20.8.16.20-lts.md new file mode 100644 index 00000000000..14c59f434d0 --- /dev/null +++ b/docs/changelogs/v20.8.16.20-lts.md @@ -0,0 +1,6 @@ +### ClickHouse release v20.8.16.20-lts FIXME as compared to v20.8.15.11-lts + +#### Bug Fix +* Backported in [#22091](https://github.com/ClickHouse/ClickHouse/issues/22091): In case if query has constant `WHERE` condition, and setting `optimize_skip_unused_shards` enabled, all shards may be skipped and query could return incorrect empty result. [#21550](https://github.com/ClickHouse/ClickHouse/pull/21550) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#22049](https://github.com/ClickHouse/ClickHouse/issues/22049): Fix deadlock in first catboost model execution. Closes [#13832](https://github.com/ClickHouse/ClickHouse/issues/13832). [#21844](https://github.com/ClickHouse/ClickHouse/pull/21844) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v20.8.17.25-lts.md b/docs/changelogs/v20.8.17.25-lts.md new file mode 100644 index 00000000000..124e5222293 --- /dev/null +++ b/docs/changelogs/v20.8.17.25-lts.md @@ -0,0 +1,10 @@ +### ClickHouse release v20.8.17.25-lts FIXME as compared to v20.8.16.20-lts + +#### Bug Fix +* Backported in [#21341](https://github.com/ClickHouse/ClickHouse/issues/21341): Disable constant folding for subqueries on the analysis stage, when the result cannot be calculated. [#18446](https://github.com/ClickHouse/ClickHouse/pull/18446) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22698](https://github.com/ClickHouse/ClickHouse/issues/22698): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). +* Backported in [#22737](https://github.com/ClickHouse/ClickHouse/issues/22737): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Build/Testing/Packaging Improvement +* Backported in [#22819](https://github.com/ClickHouse/ClickHouse/issues/22819): Update timezones info to 2020e. [#18531](https://github.com/ClickHouse/ClickHouse/pull/18531) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v20.8.18.32-lts.md b/docs/changelogs/v20.8.18.32-lts.md new file mode 100644 index 00000000000..92fba618f4a --- /dev/null +++ b/docs/changelogs/v20.8.18.32-lts.md @@ -0,0 +1,8 @@ +### ClickHouse release v20.8.18.32-lts FIXME as compared to v20.8.17.25-lts + +#### Bug Fix +* Backported in [#22288](https://github.com/ClickHouse/ClickHouse/issues/22288): Fix null dereference with `join_use_nulls=1`. [#20344](https://github.com/ClickHouse/ClickHouse/pull/20344) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22556](https://github.com/ClickHouse/ClickHouse/issues/22556): Fix bug in partial merge join with `LowCardinality`. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#22891](https://github.com/ClickHouse/ClickHouse/issues/22891): Fix approx total rows accounting for reverse reading from MergeTree. [#22726](https://github.com/ClickHouse/ClickHouse/pull/22726) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#23172](https://github.com/ClickHouse/ClickHouse/issues/23172): Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.8.19.4-stable.md b/docs/changelogs/v20.8.19.4-stable.md new file mode 100644 index 00000000000..316d5f79d30 --- /dev/null +++ b/docs/changelogs/v20.8.19.4-stable.md @@ -0,0 +1,29 @@ +### ClickHouse release v20.8.19.4-stable FIXME as compared to v20.8.18.32-lts + +#### Improvement +* Backported in [#24720](https://github.com/ClickHouse/ClickHouse/issues/24720): If tuple of NULLs, e.g. `(NULL, NULL)` is on the left hand side of `IN` operator with tuples of non-NULLs on the right hand side, e.g. `SELECT (NULL, NULL) IN ((0, 0), (3, 1))` return 0 instead of throwing an exception about incompatible types. The expression may also appear due to optimization of something like `SELECT (NULL, NULL) = (8, 0) OR (NULL, NULL) = (3, 2) OR (NULL, NULL) = (0, 0) OR (NULL, NULL) = (3, 1)`. This closes [#22017](https://github.com/ClickHouse/ClickHouse/issues/22017). [#22063](https://github.com/ClickHouse/ClickHouse/pull/22063) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#24713](https://github.com/ClickHouse/ClickHouse/issues/24713): Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24715](https://github.com/ClickHouse/ClickHouse/issues/24715): Fixed `Cannot unlink file` error on unsuccessful creation of ReplicatedMergeTree table with multidisk configuration. This closes [#21755](https://github.com/ClickHouse/ClickHouse/issues/21755). [#23433](https://github.com/ClickHouse/ClickHouse/pull/23433) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24717](https://github.com/ClickHouse/ClickHouse/issues/24717): When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#24744](https://github.com/ClickHouse/ClickHouse/issues/24744): Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#24711](https://github.com/ClickHouse/ClickHouse/issues/24711): Fix misinterpretation of some `LIKE` expressions with escape sequences. [#23610](https://github.com/ClickHouse/ClickHouse/pull/23610) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#24709](https://github.com/ClickHouse/ClickHouse/issues/24709): Fixed a bug in recovery of staled `ReplicatedMergeTree` replica. Some metadata updates could be ignored by staled replica if `ALTER` query was executed during downtime of the replica. [#23742](https://github.com/ClickHouse/ClickHouse/pull/23742) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24708](https://github.com/ClickHouse/ClickHouse/issues/24708): Fix SIGSEGV for external GROUP BY and overflow row (i.e. queries like `SELECT FROM GROUP BY WITH TOTALS SETTINGS max_bytes_before_external_group_by>0, max_rows_to_group_by>0, group_by_overflow_mode='any', totals_mode='before_having'`). [#23962](https://github.com/ClickHouse/ClickHouse/pull/23962) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#24707](https://github.com/ClickHouse/ClickHouse/issues/24707): Fix crash in MergeJoin, close [#24010](https://github.com/ClickHouse/ClickHouse/issues/24010). [#24013](https://github.com/ClickHouse/ClickHouse/pull/24013) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#24760](https://github.com/ClickHouse/ClickHouse/issues/24760): Fix race condition which could happen in RBAC under a heavy load. This PR fixes [#24090](https://github.com/ClickHouse/ClickHouse/issues/24090), [#24134](https://github.com/ClickHouse/ClickHouse/issues/24134),. [#24176](https://github.com/ClickHouse/ClickHouse/pull/24176) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#24704](https://github.com/ClickHouse/ClickHouse/issues/24704): Fixed a bug in moving Materialized View from Ordinary to Atomic database (`RENAME TABLE` query). Now inner table is moved to new database together with Materialized View. Fixes [#23926](https://github.com/ClickHouse/ClickHouse/issues/23926). [#24309](https://github.com/ClickHouse/ClickHouse/pull/24309) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24823](https://github.com/ClickHouse/ClickHouse/issues/24823): - Fixed the deadlock that can happen during LDAP role (re)mapping, when LDAP group is mapped to a nonexistent local role. [#24431](https://github.com/ClickHouse/ClickHouse/pull/24431) ([Denis Glazachev](https://github.com/traceon)). +* Backported in [#25575](https://github.com/ClickHouse/ClickHouse/issues/25575): Fix incorrect monotonicity of toWeek function. This fixes [#24422](https://github.com/ClickHouse/ClickHouse/issues/24422) . This bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/5212 , and was exposed later by smarter partition pruner. [#24446](https://github.com/ClickHouse/ClickHouse/pull/24446) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#24701](https://github.com/ClickHouse/ClickHouse/issues/24701): Fixed the behavior when query `SYSTEM RESTART REPLICA` or `SYSTEM SYNC REPLICA` is being processed infinitely. This was detected on server with extremely little amount of RAM. [#24457](https://github.com/ClickHouse/ClickHouse/pull/24457) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#24852](https://github.com/ClickHouse/ClickHouse/issues/24852): Fix bug when exception `Mutation was killed` can be thrown to the client on mutation wait when mutation not loaded into memory yet. [#24809](https://github.com/ClickHouse/ClickHouse/pull/24809) ([alesapin](https://github.com/alesapin)). +* Backported in [#25184](https://github.com/ClickHouse/ClickHouse/issues/25184): Fixed bug with declaring S3 disk at root of bucket. Earlier, it reported an error: ``` [heather] 2021.05.10 02:11:11.932234 [ 72790 ] {2ff80b7b-ec53-41cb-ac35-19bb390e1759} executeQuery: Code: 36, e.displayText() = DB::Exception: Key name is empty in path style S3 URI: (http://172.17.0.2/bucket/) (version 21.6.1.1) (from 127.0.0.1:47994) (in query: SELECT policy_name FROM system.storage_policies), Stack trace (when copying this message, always include the lines below):. [#24898](https://github.com/ClickHouse/ClickHouse/pull/24898) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#24951](https://github.com/ClickHouse/ClickHouse/issues/24951): Fix possible heap-buffer-overflow in Arrow. [#24922](https://github.com/ClickHouse/ClickHouse/pull/24922) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#25209](https://github.com/ClickHouse/ClickHouse/issues/25209): Fix crash in query with cross join and `joined_subquery_requires_alias = 0`. Fixes [#24011](https://github.com/ClickHouse/ClickHouse/issues/24011). [#25082](https://github.com/ClickHouse/ClickHouse/pull/25082) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25350](https://github.com/ClickHouse/ClickHouse/issues/25350): Fix TOCTOU error in installation script. [#25277](https://github.com/ClickHouse/ClickHouse/pull/25277) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#25354](https://github.com/ClickHouse/ClickHouse/issues/25354): Fix Logical Error Cannot sum Array/Tuple in min/maxMap. [#25298](https://github.com/ClickHouse/ClickHouse/pull/25298) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#25503](https://github.com/ClickHouse/ClickHouse/issues/25503): Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#25713](https://github.com/ClickHouse/ClickHouse/issues/25713): Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#26145](https://github.com/ClickHouse/ClickHouse/issues/26145): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.8.2.3-stable.md b/docs/changelogs/v20.8.2.3-stable.md new file mode 100644 index 00000000000..9dda8b499c1 --- /dev/null +++ b/docs/changelogs/v20.8.2.3-stable.md @@ -0,0 +1,143 @@ +### ClickHouse release v20.8.2.3-stable FIXME as compared to v20.7.1.4310-prestable + +#### Backward Incompatible Change +* Remove support for the `ODBCDriver` input/output format. This was a deprecated format once used for communication with the ClickHouse ODBC driver, now long superseded by the `ODBCDriver2` format. Resolves [#13629](https://github.com/ClickHouse/ClickHouse/issues/13629). [#13847](https://github.com/ClickHouse/ClickHouse/pull/13847) ([hexiaoting](https://github.com/hexiaoting)). +* Extend `parallel_distributed_insert_select` setting, adding an option to run `INSERT` into local table. The setting changes type from `Bool` to `UInt64`, so the values `false` and `true` are no longer supported. If you have these values in server configuration, the server will not start. Please replace them with `0` and `1`, respectively. [#14060](https://github.com/ClickHouse/ClickHouse/pull/14060) ([Azat Khuzhin](https://github.com/azat)). +* Now `OPTIMIZE FINAL` query doesn't recalculate TTL for parts that were added before TTL was created. Use `ALTER TABLE ... MATERIALIZE TTL` once to calculate them, after that `OPTIMIZE FINAL` will evaluate TTL's properly. This behavior never worked for replicated tables. [#14220](https://github.com/ClickHouse/ClickHouse/pull/14220) ([alesapin](https://github.com/alesapin)). + +#### New Feature +* Support `MaterializeMySQL` database engine. Implements [#4006](https://github.com/ClickHouse/ClickHouse/issues/4006). [#10851](https://github.com/ClickHouse/ClickHouse/pull/10851) ([Winter Zhang](https://github.com/zhang2014)). +* Support Kerberos authentication in Kafka, using `krb5` and `cyrus-sasl` libraries. [#12771](https://github.com/ClickHouse/ClickHouse/pull/12771) ([Ilya Golshtein](https://github.com/ilejn)). +* Add types `Int128`, `Int256`, `UInt256` and related functions for them. Extend Decimals with Decimal256 (precision up to 76 digits). New types are under the setting `allow_experimental_bigint_types`. [#13097](https://github.com/ClickHouse/ClickHouse/pull/13097) ([Artem Zuikov](https://github.com/4ertus2)). +* Function `position` now supports optional `start_pos` argument. [#13237](https://github.com/ClickHouse/ClickHouse/pull/13237) ([Vladimir C](https://github.com/vdimir)). +* Add `ALTER SAMPLE BY` statement that allows to change table sample clause. [#13280](https://github.com/ClickHouse/ClickHouse/pull/13280) ([Amos Bird](https://github.com/amosbird)). +* Add new optional section to the main config. [#13425](https://github.com/ClickHouse/ClickHouse/pull/13425) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add function `normalizeQuery` that replaces literals, sequences of literals and complex aliases with placeholders. Add function `normalizedQueryHash` that returns identical 64bit hash values for similar queries. It helps to analyze query log. This closes [#11271](https://github.com/ClickHouse/ClickHouse/issues/11271). [#13816](https://github.com/ClickHouse/ClickHouse/pull/13816) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `quantileExactLow` and `quantileExactHigh` implementations with respective aliases for `medianExactLow` and `medianExactHigh`. [#13818](https://github.com/ClickHouse/ClickHouse/pull/13818) ([Bharat Nallan](https://github.com/bharatnc)). +* Add function `defaultValueOfTypeName` that returns the default value for a given type. [#13877](https://github.com/ClickHouse/ClickHouse/pull/13877) ([hcz](https://github.com/hczhcz)). +* Added `date_trunc` function that truncates a date/time value to a specified date/time part. [#13888](https://github.com/ClickHouse/ClickHouse/pull/13888) ([vladimir-golovchenko](https://github.com/vladimir-golovchenko)). +* Add the ability to specify `Default` compression codec for columns that correspond to settings specified in `config.xml`. Implements: [#9074](https://github.com/ClickHouse/ClickHouse/issues/9074). [#14049](https://github.com/ClickHouse/ClickHouse/pull/14049) ([alesapin](https://github.com/alesapin)). +* Add setting `min_index_granularity_bytes` that protects against accidentally creating a table with very low `index_granularity_bytes` setting. [#14139](https://github.com/ClickHouse/ClickHouse/pull/14139) ([Bharat Nallan](https://github.com/bharatnc)). +* Add `countDigits(x)` function that count number of decimal digits in integer or decimal column. Add `isDecimalOverflow(d, [p])` function that checks if the value in Decimal column is out of its (or specified) precision. [#14151](https://github.com/ClickHouse/ClickHouse/pull/14151) ([Artem Zuikov](https://github.com/4ertus2)). + +#### Performance Improvement +* When performing trivial `INSERT SELECT` queries, automatically set `max_threads` to 1 or `max_insert_threads`, and set `max_block_size` to `min_insert_block_size_rows`. Related to [#5907](https://github.com/ClickHouse/ClickHouse/issues/5907). [#12195](https://github.com/ClickHouse/ClickHouse/pull/12195) ([flynn](https://github.com/ucasfl)). +* Optimize `has()`, `indexOf()` and `countEqual()` functions for `Array(LowCardinality(T))` and constant right arguments. [#12550](https://github.com/ClickHouse/ClickHouse/pull/12550) ([Mike Kot](https://github.com/myrrc)). +* Slightly improve performance of aggregation by UInt8/UInt16 keys. [#13099](https://github.com/ClickHouse/ClickHouse/pull/13099) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fail fast if `max_rows_to_read` limit is exceeded on parts scan. The motivation behind this change is to skip ranges scan for all selected parts if it is clear that `max_rows_to_read` is already exceeded. The change is quite noticeable for queries over big number of parts. [#13677](https://github.com/ClickHouse/ClickHouse/pull/13677) ([Roman Khavronenko](https://github.com/hagen1778)). +* Enable parallel INSERTs for table engines `Null`, `Memory`, `Distributed` and `Buffer`. [#14120](https://github.com/ClickHouse/ClickHouse/pull/14120) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Slightly optimize very short queries with LowCardinality. [#14129](https://github.com/ClickHouse/ClickHouse/pull/14129) ([Anton Popov](https://github.com/CurtizJ)). + +#### Improvement +* Added cache layer for DiskS3 (cache to local disk mark and index files). [#13076](https://github.com/ClickHouse/ClickHouse/pull/13076) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix assert when decimal has too large negative exponent. Fixes [#13188](https://github.com/ClickHouse/ClickHouse/issues/13188). [#13228](https://github.com/ClickHouse/ClickHouse/pull/13228) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add QueryTimeMicroseconds, SelectQueryTimeMicroseconds and InsertQueryTimeMicroseconds to system.events. [#13336](https://github.com/ClickHouse/ClickHouse/pull/13336) ([ianton-ru](https://github.com/ianton-ru)). +* Proper remote host checking in S3 redirects (security-related thing). [#13404](https://github.com/ClickHouse/ClickHouse/pull/13404) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Enable mixed granularity parts by default. [#13449](https://github.com/ClickHouse/ClickHouse/pull/13449) ([alesapin](https://github.com/alesapin)). +* Move parts from DIskLocal to DiskS3 in parallel. [#13459](https://github.com/ClickHouse/ClickHouse/pull/13459) ([Pavel Kovalenko](https://github.com/Jokser)). +* Support compound identifiers for custom settings. [#13496](https://github.com/ClickHouse/ClickHouse/pull/13496) ([Vitaly Baranov](https://github.com/vitlibar)). +* Provide monotonicity for `toDate/toDateTime` functions in more cases. Now the input arguments are saturated more naturally and provides better monotonicity. [#13497](https://github.com/ClickHouse/ClickHouse/pull/13497) ([Amos Bird](https://github.com/amosbird)). +* In previous versions `lcm` function may produce assertion violation in debug build if called with specifically crafted arguments. This fixes [#13368](https://github.com/ClickHouse/ClickHouse/issues/13368). [#13510](https://github.com/ClickHouse/ClickHouse/pull/13510) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add MergeTree Write-Ahead-Log(WAL) dump tool. [#13640](https://github.com/ClickHouse/ClickHouse/pull/13640) ([BohuTANG](https://github.com/BohuTANG)). +* Backported in [#14360](https://github.com/ClickHouse/ClickHouse/issues/14360): Added Redis requirepass authorization. [#13688](https://github.com/ClickHouse/ClickHouse/pull/13688) ([Ivan Torgashov](https://github.com/it1804)). +* Avoid too slow queries when arrays are manipulated as fields. Throw exception instead. [#13753](https://github.com/ClickHouse/ClickHouse/pull/13753) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* 1. Add [GTID-Based Replication](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html), it works even when replication topology changes, and supported/prefered in MySQL 5.6/5.7/8.0 2. Add BIT/SET filed type supports 3. Fix up varchar type meta length bug. [#13820](https://github.com/ClickHouse/ClickHouse/pull/13820) ([BohuTANG](https://github.com/BohuTANG)). +* Fix data race in `lgamma` function. This race was caught only in `tsan`, no side effects a really happened. [#13842](https://github.com/ClickHouse/ClickHouse/pull/13842) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Function `arrayCompact` will compare NaNs bitwise if the type of array elements is Float32/Float64. In previous versions NaNs were always not equal if the type of array elements is Float32/Float64 and were always equal if the type is more complex, like Nullable(Float64). This closes [#13857](https://github.com/ClickHouse/ClickHouse/issues/13857). [#13868](https://github.com/ClickHouse/ClickHouse/pull/13868) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better error message for null value of TabSeparatedRow format. [#13906](https://github.com/ClickHouse/ClickHouse/pull/13906) ([jiang tao](https://github.com/tomjiang1987)). +* Fix wrong error for long queries. It was possible to get syntax error other than `Max query size exceeded` for correct query. [#13928](https://github.com/ClickHouse/ClickHouse/pull/13928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Corrected an error in AvroConfluent format that caused the Kafka table engine to stop processing messages when an abnormally small, malformed, message was received. [#13941](https://github.com/ClickHouse/ClickHouse/pull/13941) ([Gervasio Varela](https://github.com/gervarela)). +* Increase limit in -Resample combinator to 1M. [#13947](https://github.com/ClickHouse/ClickHouse/pull/13947) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Conditional aggregate functions (for example: `avgIf`, `sumIf`, `maxIf`) should return `NULL` when miss rows and use nullable arguments. [#13964](https://github.com/ClickHouse/ClickHouse/pull/13964) ([Winter Zhang](https://github.com/zhang2014)). +* Slightly better performance of Memory table if it was constructed from a huge number of very small blocks (that's unlikely). Author of the idea: [Mark Papadakis](https://github.com/markpapadakis). Closes [#14043](https://github.com/ClickHouse/ClickHouse/issues/14043). [#14056](https://github.com/ClickHouse/ClickHouse/pull/14056) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now it's possible to `ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'zk://:/path-in-zookeeper'`. It's useful for shipping data to new clusters. [#14155](https://github.com/ClickHouse/ClickHouse/pull/14155) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix +* subquery hash values are not enough to distinguish. [#8333](https://github.com/ClickHouse/ClickHouse/issues/8333). [#8367](https://github.com/ClickHouse/ClickHouse/pull/8367) ([Amos Bird](https://github.com/amosbird)). +* Removed wrong auth access check when using ClickHouseDictionarySource to query remote tables. [#12756](https://github.com/ClickHouse/ClickHouse/pull/12756) ([sundyli](https://github.com/sundy-li)). +* Fix access to redis dictionary after connection was dropped once. It may happen with `cache` and `direct` dictionary layouts. [#13082](https://github.com/ClickHouse/ClickHouse/pull/13082) ([Anton Popov](https://github.com/CurtizJ)). +* Fix parsing row policies from users.xml when names of databases or tables contain dots. This fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779), [#12527](https://github.com/ClickHouse/ClickHouse/issues/12527). [#13199](https://github.com/ClickHouse/ClickHouse/pull/13199) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix missing or excessive headers in `TSV/CSVWithNames` formats. This fixes [#12504](https://github.com/ClickHouse/ClickHouse/issues/12504). [#13343](https://github.com/ClickHouse/ClickHouse/pull/13343) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible race in `StorageMemory`. https://clickhouse-test-reports.s3.yandex.net/0/9cac8a7244063d2092ad25d45502611e18d3749c/stress_test_(thread)/stderr.log Have no idea how to write a test. [#13416](https://github.com/ClickHouse/ClickHouse/pull/13416) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix wrong code in function `netloc`. This fixes [#13335](https://github.com/ClickHouse/ClickHouse/issues/13335). [#13446](https://github.com/ClickHouse/ClickHouse/pull/13446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix premature `ON CLUSTER` timeouts for queries that must be executed on a single replica. Fixes [#6704](https://github.com/ClickHouse/ClickHouse/issues/6704), Fixes [#7228](https://github.com/ClickHouse/ClickHouse/issues/7228), Fixes [#13361](https://github.com/ClickHouse/ClickHouse/issues/13361), Fixes [#11884](https://github.com/ClickHouse/ClickHouse/issues/11884). [#13450](https://github.com/ClickHouse/ClickHouse/pull/13450) ([alesapin](https://github.com/alesapin)). +* Fix secondary indices corruption in compact parts. [#13538](https://github.com/ClickHouse/ClickHouse/pull/13538) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed the behaviour when sometimes cache-dictionary returned default value instead of present value from source. [#13624](https://github.com/ClickHouse/ClickHouse/pull/13624) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Concurrent `ALTER ... REPLACE/MOVE PARTITION ...` queries might cause deadlock. It's fixed. [#13626](https://github.com/ClickHouse/ClickHouse/pull/13626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix typo in error message about `The value of 'number_of_free_entries_in_pool_to_lower_max_size_of_merge' setting`. [#13678](https://github.com/ClickHouse/ClickHouse/pull/13678) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix crash in JOIN with StorageMerge and `set enable_optimize_predicate_expression=1`. [#13679](https://github.com/ClickHouse/ClickHouse/pull/13679) ([Artem Zuikov](https://github.com/4ertus2)). +* Do not optimize any(arrayJoin()) -> arrayJoin() under optimize_move_functions_out_of_any. [#13681](https://github.com/ClickHouse/ClickHouse/pull/13681) ([Azat Khuzhin](https://github.com/azat)). +* Fix visible data clobbering by progress bar in client in interactive mode. This fixes [#12562](https://github.com/ClickHouse/ClickHouse/issues/12562) and [#13369](https://github.com/ClickHouse/ClickHouse/issues/13369) and [#13584](https://github.com/ClickHouse/ClickHouse/issues/13584) and fixes [#12964](https://github.com/ClickHouse/ClickHouse/issues/12964). [#13691](https://github.com/ClickHouse/ClickHouse/pull/13691) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix incorrect message in `clickhouse-server.init` while checking user and group. [#13711](https://github.com/ClickHouse/ClickHouse/pull/13711) ([ylchou](https://github.com/ylchou)). +* Fixes /replicas_status endpoint response status code when verbose=1. [#13722](https://github.com/ClickHouse/ClickHouse/pull/13722) ([javi santana](https://github.com/javisantana)). +* Fix logging Settings.Names/Values when log_queries_min_type > QUERY_START. [#13737](https://github.com/ClickHouse/ClickHouse/pull/13737) ([Azat Khuzhin](https://github.com/azat)). +* Fix race condition between DETACH and background merges. Parts may revive after detach. This is continuation of [#8602](https://github.com/ClickHouse/ClickHouse/issues/8602) that did not fix the issue but introduced a test that started to fail in very rare cases, demonstrating the issue. [#13746](https://github.com/ClickHouse/ClickHouse/pull/13746) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add range check for h3KRing function. This fixes [#13633](https://github.com/ClickHouse/ClickHouse/issues/13633). [#13752](https://github.com/ClickHouse/ClickHouse/pull/13752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed `Directory not empty` error when concurrently executing `DROP DATABASE` and `CREATE TABLE`. [#13756](https://github.com/ClickHouse/ClickHouse/pull/13756) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix step overflow in range(). [#13790](https://github.com/ClickHouse/ClickHouse/pull/13790) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#14387](https://github.com/ClickHouse/ClickHouse/issues/14387): Fix arrayJoin() capturing in lambda (LOGICAL_ERROR). [#13792](https://github.com/ClickHouse/ClickHouse/pull/13792) ([Azat Khuzhin](https://github.com/azat)). +* Fix reading from MergeTree table with INDEX of type SET fails when comparing against NULL. This fixes [#13686](https://github.com/ClickHouse/ClickHouse/issues/13686). [#13793](https://github.com/ClickHouse/ClickHouse/pull/13793) ([Amos Bird](https://github.com/amosbird)). +* Fix topK/topKWeighted merge (with non-default parameters). [#13817](https://github.com/ClickHouse/ClickHouse/pull/13817) ([Azat Khuzhin](https://github.com/azat)). +* Fix incorrect sorting for `FixedString` columns. Fixes [#13182](https://github.com/ClickHouse/ClickHouse/issues/13182). [#13887](https://github.com/ClickHouse/ClickHouse/pull/13887) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed potential deadlock when renaming `Distributed` table. [#13922](https://github.com/ClickHouse/ClickHouse/pull/13922) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix wrong results in select queries with `DISTINCT` keyword in case `optimize_duplicate_order_by_and_distinct` setting is enabled. [#13925](https://github.com/ClickHouse/ClickHouse/pull/13925) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix parser to reject create table as table function with engine. [#13940](https://github.com/ClickHouse/ClickHouse/pull/13940) ([hcz](https://github.com/hczhcz)). +* Backported in [#14389](https://github.com/ClickHouse/ClickHouse/issues/14389): Fix GRANT ALL statement when executed on a non-global level. [#13987](https://github.com/ClickHouse/ClickHouse/pull/13987) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed wrong mount point in extra info for `Poco::Exception: no space left on device`. [#14050](https://github.com/ClickHouse/ClickHouse/pull/14050) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix pointInPolygon with const 2d array as polygon. [#14079](https://github.com/ClickHouse/ClickHouse/pull/14079) ([Alexey Ilyukhov](https://github.com/livace)). +* Backported in [#14311](https://github.com/ClickHouse/ClickHouse/issues/14311): Fix DistributedFilesToInsert metric (zeroed when it should not). [#14095](https://github.com/ClickHouse/ClickHouse/pull/14095) ([Azat Khuzhin](https://github.com/azat)). +* When waiting for a dictionary update to complete, use the timeout specified by `query_wait_timeout_milliseconds` setting instead of a hard-coded value. [#14105](https://github.com/ClickHouse/ClickHouse/pull/14105) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix formatting of minimal negative decimal numbers. This fixes [#14111](https://github.com/ClickHouse/ClickHouse/issues/14111). [#14119](https://github.com/ClickHouse/ClickHouse/pull/14119) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix creation of tables with named tuples. This fixes [#13027](https://github.com/ClickHouse/ClickHouse/issues/13027). [#14143](https://github.com/ClickHouse/ClickHouse/pull/14143) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#14312](https://github.com/ClickHouse/ClickHouse/issues/14312): Fix crash when INERT INTO Kafka engine table from an empty result set with a subquery. ... [#14203](https://github.com/ClickHouse/ClickHouse/pull/14203) ([Dongdong Yang](https://github.com/donge)). +* Fixed incorrect sorting order if LowCardinality column. This fixes [#13958](https://github.com/ClickHouse/ClickHouse/issues/13958). [#14223](https://github.com/ClickHouse/ClickHouse/pull/14223) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix crash in mark inclusion search introduced in https://github.com/ClickHouse/ClickHouse/pull/12277 . [#14225](https://github.com/ClickHouse/ClickHouse/pull/14225) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#14310](https://github.com/ClickHouse/ClickHouse/issues/14310): fixes [#14231](https://github.com/ClickHouse/ClickHouse/issues/14231) fix wrong lexer in MaterializeMySQL database engine dump stage. [#14232](https://github.com/ClickHouse/ClickHouse/pull/14232) ([Winter Zhang](https://github.com/zhang2014)). +* Fix handling of empty transactions in `MaterializeMySQL` database engine. This fixes [#14235](https://github.com/ClickHouse/ClickHouse/issues/14235). [#14253](https://github.com/ClickHouse/ClickHouse/pull/14253) ([BohuTANG](https://github.com/BohuTANG)). +* Backported in [#14332](https://github.com/ClickHouse/ClickHouse/issues/14332): Disallows `CODEC` on `ALIAS` column type. Fixes [#13911](https://github.com/ClickHouse/ClickHouse/issues/13911). [#14263](https://github.com/ClickHouse/ClickHouse/pull/14263) ([Bharat Nallan](https://github.com/bharatnc)). +* Backported in [#14308](https://github.com/ClickHouse/ClickHouse/issues/14308): Fix segfault in `clickhouse-odbc-bridge` during schema fetch from some external sources. This PR fixes [#13861](https://github.com/ClickHouse/ClickHouse/issues/13861). [#14267](https://github.com/ClickHouse/ClickHouse/pull/14267) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#14410](https://github.com/ClickHouse/ClickHouse/issues/14410): Fix QueryPlan lifetime (for EXPLAIN PIPELINE graph=1) for queries with nested interpreter. [#14315](https://github.com/ClickHouse/ClickHouse/pull/14315) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#14396](https://github.com/ClickHouse/ClickHouse/issues/14396): Fix exception during ALTER LIVE VIEW query with REFRESH command. [#14320](https://github.com/ClickHouse/ClickHouse/pull/14320) ([Bharat Nallan](https://github.com/bharatnc)). +* Backported in [#14343](https://github.com/ClickHouse/ClickHouse/issues/14343): Fix crash during `ALTER` query for table which was created `AS table_function`. Fixes [#14212](https://github.com/ClickHouse/ClickHouse/issues/14212). [#14326](https://github.com/ClickHouse/ClickHouse/pull/14326) ([alesapin](https://github.com/alesapin)). +* Backported in [#14363](https://github.com/ClickHouse/ClickHouse/issues/14363): Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. [#14334](https://github.com/ClickHouse/ClickHouse/pull/14334) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#14506](https://github.com/ClickHouse/ClickHouse/issues/14506): Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. Continuation of [#14334](https://github.com/ClickHouse/ClickHouse/issues/14334). [#14402](https://github.com/ClickHouse/ClickHouse/pull/14402) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#14487](https://github.com/ClickHouse/ClickHouse/issues/14487): Fix bug which leads to wrong merges assignment if table has partitions with a single part. [#14444](https://github.com/ClickHouse/ClickHouse/pull/14444) ([alesapin](https://github.com/alesapin)). +* Backported in [#14481](https://github.com/ClickHouse/ClickHouse/issues/14481): Check for array size overflow in `topK` aggregate function. Without this check the user may send a query with carefully crafter parameters that will lead to server crash. This closes [#14452](https://github.com/ClickHouse/ClickHouse/issues/14452). [#14467](https://github.com/ClickHouse/ClickHouse/pull/14467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Build/Testing/Packaging Improvement +* Move Dockerfiles from integration tests to `docker/test` directory. docker_compose files are available in `runner` docker container. Docker images are built in CI and not in integration tests. [#13448](https://github.com/ClickHouse/ClickHouse/pull/13448) ([Ilya Yatsishin](https://github.com/qoega)). +* Skip PR's from robot-clickhouse. [#13489](https://github.com/ClickHouse/ClickHouse/pull/13489) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix typos in code with codespell. [#13511](https://github.com/ClickHouse/ClickHouse/pull/13511) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable check for typos in code with `codespell`. [#13513](https://github.com/ClickHouse/ClickHouse/pull/13513) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Ensure that there is no copy-pasted GPL code. [#13514](https://github.com/ClickHouse/ClickHouse/pull/13514) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to run `clickhouse` binary without configuration. [#13515](https://github.com/ClickHouse/ClickHouse/pull/13515) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added `clickhouse install` script, that is useful if you only have a single binary. [#13528](https://github.com/ClickHouse/ClickHouse/pull/13528) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix the remaining shellcheck notices. A preparation to enable Shellcheck. [#13529](https://github.com/ClickHouse/ClickHouse/pull/13529) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable Shellcheck in CI as a linter of .sh tests. This closes [#13168](https://github.com/ClickHouse/ClickHouse/issues/13168). [#13530](https://github.com/ClickHouse/ClickHouse/pull/13530) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make sure [#10977](https://github.com/ClickHouse/ClickHouse/issues/10977) is invalid. [#13539](https://github.com/ClickHouse/ClickHouse/pull/13539) ([Amos Bird](https://github.com/amosbird)). +* Increasing health-check timeouts for ClickHouse nodes and adding support to dump docker-compose logs if unhealthy containers found. [#13612](https://github.com/ClickHouse/ClickHouse/pull/13612) ([vzakaznikov](https://github.com/vzakaznikov)). +* Build ClickHouse with the most fresh tzdata from package repository. [#13623](https://github.com/ClickHouse/ClickHouse/pull/13623) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Removed `-DENABLE_CURL_CLIENT` for `contrib/aws`. [#13628](https://github.com/ClickHouse/ClickHouse/pull/13628) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Expose version of embedded tzdata via TZDATA_VERSION in system.build_options. [#13648](https://github.com/ClickHouse/ClickHouse/pull/13648) ([filimonov](https://github.com/filimonov)). +* Updating LDAP user authentication suite to check that it works with RBAC. [#13656](https://github.com/ClickHouse/ClickHouse/pull/13656) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add a CMake option to fail configuration instead of auto-reconfiguration, enabled by default. [#13687](https://github.com/ClickHouse/ClickHouse/pull/13687) ([Konstantin](https://github.com/podshumok)). +* Fix link error in shared build. [#13700](https://github.com/ClickHouse/ClickHouse/pull/13700) ([Amos Bird](https://github.com/amosbird)). +* FIx cassandra build on Mac OS. [#13708](https://github.com/ClickHouse/ClickHouse/pull/13708) ([Ilya Yatsishin](https://github.com/qoega)). +* Added docker image for style check. Added style check that all docker and docker compose files are located in docker directory. [#13724](https://github.com/ClickHouse/ClickHouse/pull/13724) ([Ilya Yatsishin](https://github.com/qoega)). +* ZooKeeper cannot work reliably in unit tests in CI infrastructure. Using unit tests for ZooKeeper interaction with real ZooKeeper is bad idea from the start (unit tests are not supposed to verify complex distributed systems). We already using integration tests for this purpose and they are better suited. [#13745](https://github.com/ClickHouse/ClickHouse/pull/13745) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Testflows LDAP module: adding missing certificates and dhparam.pem for openldap4. [#13780](https://github.com/ClickHouse/ClickHouse/pull/13780) ([vzakaznikov](https://github.com/vzakaznikov)). +* Enabled text-log in stress test to find more bugs. [#13855](https://github.com/ClickHouse/ClickHouse/pull/13855) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* * Adding retry logic when bringing up docker-compose cluster * Increasing COMPOSE_HTTP_TIMEOUT. [#14112](https://github.com/ClickHouse/ClickHouse/pull/14112) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add the ability to write js-style comments in skip_list.json. [#14159](https://github.com/ClickHouse/ClickHouse/pull/14159) ([alesapin](https://github.com/alesapin)). +* Switch tests docker images to use test-base parent. [#14167](https://github.com/ClickHouse/ClickHouse/pull/14167) ([Ilya Yatsishin](https://github.com/qoega)). +* Actually there are no symlinks there, so `-type f` is enough ``` ~/workspace/ClickHouse/contrib/cctz/testdata/zoneinfo$ find . -type l -ls | wc -l 0 ``` Closes [#14209](https://github.com/ClickHouse/ClickHouse/issues/14209). [#14215](https://github.com/ClickHouse/ClickHouse/pull/14215) ([filimonov](https://github.com/filimonov)). + +#### Other +* Fix readline so it dumps history to file now. [#13600](https://github.com/ClickHouse/ClickHouse/pull/13600) ([Amos Bird](https://github.com/amosbird)). +* Create `system` database with `Atomic` engine by default. [#13680](https://github.com/ClickHouse/ClickHouse/pull/13680) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Bump pymdown-extensions from 7.1 to 8.0 in /docs/tools'. [#13645](https://github.com/ClickHouse/ClickHouse/pull/13645) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Bump protobuf from 3.12.4 to 3.13.0 in /docs/tools'. [#13824](https://github.com/ClickHouse/ClickHouse/pull/13824) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). + diff --git a/docs/changelogs/v20.8.3.18-stable.md b/docs/changelogs/v20.8.3.18-stable.md new file mode 100644 index 00000000000..92b86dd99e7 --- /dev/null +++ b/docs/changelogs/v20.8.3.18-stable.md @@ -0,0 +1,16 @@ +### ClickHouse release v20.8.3.18-stable FIXME as compared to v20.8.2.3-stable + +#### Improvement +* Backported in [#14874](https://github.com/ClickHouse/ClickHouse/issues/14874): Allow using multi-volume storage configuration in storage Distributed. [#14839](https://github.com/ClickHouse/ClickHouse/pull/14839) ([Pavel Kovalenko](https://github.com/Jokser)). +* Speed up server shutdown process if there are ongoing S3 requests. [#14858](https://github.com/ClickHouse/ClickHouse/pull/14858) ([Pavel Kovalenko](https://github.com/Jokser)). + +#### Bug Fix +* Backported in [#14600](https://github.com/ClickHouse/ClickHouse/issues/14600): Fix rare segfaults in functions with combinator -Resample, which could appear in result of overflow with very large parameters. [#14562](https://github.com/ClickHouse/ClickHouse/pull/14562) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#14727](https://github.com/ClickHouse/ClickHouse/issues/14727): Cleanup data directory after Zookeeper exceptions during CreateQuery for StorageReplicatedMergeTree Engine. [#14563](https://github.com/ClickHouse/ClickHouse/pull/14563) ([Bharat Nallan](https://github.com/bharatnc)). +* Backported in [#14653](https://github.com/ClickHouse/ClickHouse/issues/14653): Added the checker as neither calling `lc->isNullable()` nor calling `ls->getDictionaryPtr()->isNullable()` would return the correct result. [#14591](https://github.com/ClickHouse/ClickHouse/pull/14591) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#14666](https://github.com/ClickHouse/ClickHouse/issues/14666): Fix wrong Decimal multiplication result caused wrong decimal scale of result column. [#14603](https://github.com/ClickHouse/ClickHouse/pull/14603) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#14793](https://github.com/ClickHouse/ClickHouse/issues/14793): Fix bug when `ALTER UPDATE` mutation with Nullable column in assignment expression and constant value (like `UPDATE x = 42`) leads to incorrect value in column or segfault. Fixes [#13634](https://github.com/ClickHouse/ClickHouse/issues/13634), [#14045](https://github.com/ClickHouse/ClickHouse/issues/14045). [#14646](https://github.com/ClickHouse/ClickHouse/pull/14646) ([alesapin](https://github.com/alesapin)). +* Backported in [#14722](https://github.com/ClickHouse/ClickHouse/issues/14722): Fixed missed default database name in metadata of materialized view when executing `ALTER ... MODIFY QUERY`. [#14664](https://github.com/ClickHouse/ClickHouse/pull/14664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#14910](https://github.com/ClickHouse/ClickHouse/issues/14910): Fix SIGSEGV for an attempt to INSERT into StorageFile(fd). [#14887](https://github.com/ClickHouse/ClickHouse/pull/14887) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#14943](https://github.com/ClickHouse/ClickHouse/issues/14943): Fix the issue when some invocations of `extractAllGroups` function may trigger "Memory limit exceeded" error. This fixes [#13383](https://github.com/ClickHouse/ClickHouse/issues/13383). [#14889](https://github.com/ClickHouse/ClickHouse/pull/14889) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.8.4.11-lts.md b/docs/changelogs/v20.8.4.11-lts.md new file mode 100644 index 00000000000..ea8b71e1992 --- /dev/null +++ b/docs/changelogs/v20.8.4.11-lts.md @@ -0,0 +1,38 @@ +### ClickHouse release v20.8.4.11-lts FIXME as compared to v20.8.3.18-stable + +#### Improvement +* Backported in [#15569](https://github.com/ClickHouse/ClickHouse/issues/15569): Now it's possible to change the type of version column for `VersionedCollapsingMergeTree` with `ALTER` query. [#15442](https://github.com/ClickHouse/ClickHouse/pull/15442) ([alesapin](https://github.com/alesapin)). + +#### Bug Fix +* Backported in [#15017](https://github.com/ClickHouse/ClickHouse/issues/15017): Fixed the incorrect sorting order of `Nullable` column. This fixes [#14344](https://github.com/ClickHouse/ClickHouse/issues/14344). [#14495](https://github.com/ClickHouse/ClickHouse/pull/14495) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14823](https://github.com/ClickHouse/ClickHouse/issues/14823): Fix wrong monotonicity detection for shrunk `Int -> Int` cast of signed types. It might lead to incorrect query result. This bug is unveiled in [#14513](https://github.com/ClickHouse/ClickHouse/issues/14513). [#14783](https://github.com/ClickHouse/ClickHouse/pull/14783) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15148](https://github.com/ClickHouse/ClickHouse/issues/15148): Fix a problem where the server may get stuck on startup while talking to ZooKeeper, if the configuration files have to be fetched from ZK (using the `from_zk` include option). This fixes [#14814](https://github.com/ClickHouse/ClickHouse/issues/14814). [#14843](https://github.com/ClickHouse/ClickHouse/pull/14843) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#15251](https://github.com/ClickHouse/ClickHouse/issues/15251): Fixed segfault in CacheDictionary [#14837](https://github.com/ClickHouse/ClickHouse/issues/14837). [#14879](https://github.com/ClickHouse/ClickHouse/pull/14879) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14955](https://github.com/ClickHouse/ClickHouse/issues/14955): Fixed `.metadata.tmp File exists` error when using `MaterializeMySQL` database engine. [#14898](https://github.com/ClickHouse/ClickHouse/pull/14898) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#14987](https://github.com/ClickHouse/ClickHouse/issues/14987): Publish CPU frequencies per logical core in `system.asynchronous_metrics`. This fixes [#14923](https://github.com/ClickHouse/ClickHouse/issues/14923). [#14924](https://github.com/ClickHouse/ClickHouse/pull/14924) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#14969](https://github.com/ClickHouse/ClickHouse/issues/14969): Fix to make predicate push down work when subquery contains finalizeAggregation function. Fixes [#14847](https://github.com/ClickHouse/ClickHouse/issues/14847). [#14937](https://github.com/ClickHouse/ClickHouse/pull/14937) ([filimonov](https://github.com/filimonov)). +* Backported in [#15053](https://github.com/ClickHouse/ClickHouse/issues/15053): Now settings `number_of_free_entries_in_pool_to_execute_mutation` and `number_of_free_entries_in_pool_to_lower_max_size_of_merge` can be equal to `background_pool_size`. [#14975](https://github.com/ClickHouse/ClickHouse/pull/14975) ([alesapin](https://github.com/alesapin)). +* Backported in [#15077](https://github.com/ClickHouse/ClickHouse/issues/15077): Fix crash in RIGHT or FULL JOIN with join_algorith='auto' when memory limit exceeded and we should change HashJoin with MergeJoin. [#15002](https://github.com/ClickHouse/ClickHouse/pull/15002) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15102](https://github.com/ClickHouse/ClickHouse/issues/15102): Fixed `Cannot rename ... errno: 22, strerror: Invalid argument` error on DDL query execution in Atomic database when running clickhouse-server in docker on Mac OS. [#15024](https://github.com/ClickHouse/ClickHouse/pull/15024) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15056](https://github.com/ClickHouse/ClickHouse/issues/15056): If function `bar` was called with specifically crafted arguments, buffer overflow was possible. This closes [#13926](https://github.com/ClickHouse/ClickHouse/issues/13926). [#15028](https://github.com/ClickHouse/ClickHouse/pull/15028) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15049](https://github.com/ClickHouse/ClickHouse/issues/15049): We already use padded comparison between String and FixedString (https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionsComparison.h#L333). This PR applies the same logic to field comparison which corrects the usage of FixedString as primary keys. This fixes [#14908](https://github.com/ClickHouse/ClickHouse/issues/14908). [#15033](https://github.com/ClickHouse/ClickHouse/pull/15033) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15142](https://github.com/ClickHouse/ClickHouse/issues/15142): Fixes `Data compressed with different methods` in `join_algorithm='auto'`. Keep LowCardinality as type for left table join key in `join_algorithm='partial_merge'`. [#15088](https://github.com/ClickHouse/ClickHouse/pull/15088) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15208](https://github.com/ClickHouse/ClickHouse/issues/15208): Adjust decimals field size in mysql column definition packet. [#15152](https://github.com/ClickHouse/ClickHouse/pull/15152) ([maqroll](https://github.com/maqroll)). +* Backported in [#15222](https://github.com/ClickHouse/ClickHouse/issues/15222): Fix bug in table engine `Buffer` which doesn't allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). +* Backported in [#15405](https://github.com/ClickHouse/ClickHouse/issues/15405): Fix instance crash when using joinGet with LowCardinality types. This fixes [#15214](https://github.com/ClickHouse/ClickHouse/issues/15214). [#15220](https://github.com/ClickHouse/ClickHouse/pull/15220) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15489](https://github.com/ClickHouse/ClickHouse/issues/15489): Fix 'Unknown identifier' in GROUP BY when query has JOIN over Merge table. [#15242](https://github.com/ClickHouse/ClickHouse/pull/15242) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15281](https://github.com/ClickHouse/ClickHouse/issues/15281): Fix MSan report in QueryLog. Uninitialized memory can be used for the field `memory_usage`. [#15258](https://github.com/ClickHouse/ClickHouse/pull/15258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15402](https://github.com/ClickHouse/ClickHouse/issues/15402): Fix hang of queries with a lot of subqueries to same table of `MySQL` engine. Previously, if there were more than 16 subqueries to same `MySQL` table in query, it hang forever. [#15299](https://github.com/ClickHouse/ClickHouse/pull/15299) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#15338](https://github.com/ClickHouse/ClickHouse/issues/15338): Fix rare race condition on server startup when system.logs are enabled. [#15300](https://github.com/ClickHouse/ClickHouse/pull/15300) ([alesapin](https://github.com/alesapin)). +* Backported in [#15334](https://github.com/ClickHouse/ClickHouse/issues/15334): Fix race condition during MergeTree table rename and background cleanup. [#15304](https://github.com/ClickHouse/ClickHouse/pull/15304) ([alesapin](https://github.com/alesapin)). +* Backported in [#15447](https://github.com/ClickHouse/ClickHouse/issues/15447): Report proper error when the second argument of `boundingRatio` aggregate function has a wrong type. [#15407](https://github.com/ClickHouse/ClickHouse/pull/15407) ([detailyang](https://github.com/detailyang)). +* Backported in [#15505](https://github.com/ClickHouse/ClickHouse/issues/15505): Fix bug with event subscription in DDLWorker which rarely may lead to query hangs in `ON CLUSTER`. Introduced in [#13450](https://github.com/ClickHouse/ClickHouse/issues/13450). [#15477](https://github.com/ClickHouse/ClickHouse/pull/15477) ([alesapin](https://github.com/alesapin)). +* Backported in [#15550](https://github.com/ClickHouse/ClickHouse/issues/15550): Fix `Missing columns` errors when selecting columns which absent in data, but depend on other columns which also absent in data. Fixes [#15530](https://github.com/ClickHouse/ClickHouse/issues/15530). [#15532](https://github.com/ClickHouse/ClickHouse/pull/15532) ([alesapin](https://github.com/alesapin)). +* Backported in [#15558](https://github.com/ClickHouse/ClickHouse/issues/15558): Fix bug when `ILIKE` operator stops being case insensitive if `LIKE` with the same pattern was executed. [#15536](https://github.com/ClickHouse/ClickHouse/pull/15536) ([alesapin](https://github.com/alesapin)). +* Backported in [#15726](https://github.com/ClickHouse/ClickHouse/issues/15726): Mutation might hang waiting for some non-existent part after `MOVE` or `REPLACE PARTITION` or, in rare cases, after `DETACH` or `DROP PARTITION`. It's fixed. [#15537](https://github.com/ClickHouse/ClickHouse/pull/15537) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15652](https://github.com/ClickHouse/ClickHouse/issues/15652): Fix 'Database doesn't exist.' in queries with IN and Distributed table when there's no database on initiator. [#15538](https://github.com/ClickHouse/ClickHouse/pull/15538) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15629](https://github.com/ClickHouse/ClickHouse/issues/15629): Significantly reduce memory usage in AggregatingInOrderTransform/optimize_aggregation_in_order. [#15543](https://github.com/ClickHouse/ClickHouse/pull/15543) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#15585](https://github.com/ClickHouse/ClickHouse/issues/15585): Prevent the possibility of error message `Could not calculate available disk space (statvfs), errno: 4, strerror: Interrupted system call`. This fixes [#15541](https://github.com/ClickHouse/ClickHouse/issues/15541). [#15557](https://github.com/ClickHouse/ClickHouse/pull/15557) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15666](https://github.com/ClickHouse/ClickHouse/issues/15666): Fixed `Element ... is not a constant expression` error when using `JSON*` function result in `VALUES`, `LIMIT` or right side of `IN` operator. [#15589](https://github.com/ClickHouse/ClickHouse/pull/15589) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15715](https://github.com/ClickHouse/ClickHouse/issues/15715): Fix the order of destruction for resources in `ReadFromStorage` step of query plan. It might cause crashes in rare cases. Possibly connected with [#15610](https://github.com/ClickHouse/ClickHouse/issues/15610). [#15645](https://github.com/ClickHouse/ClickHouse/pull/15645) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v20.8.5.45-lts.md b/docs/changelogs/v20.8.5.45-lts.md new file mode 100644 index 00000000000..c4043625f0d --- /dev/null +++ b/docs/changelogs/v20.8.5.45-lts.md @@ -0,0 +1,37 @@ +### ClickHouse release v20.8.5.45-lts FIXME as compared to v20.8.4.11-lts + +#### Improvement +* Backported in [#16146](https://github.com/ClickHouse/ClickHouse/issues/16146): Now it's allowed to execute `ALTER ... ON CLUSTER` queries regardless of the `` setting in cluster config. [#16075](https://github.com/ClickHouse/ClickHouse/pull/16075) ([alesapin](https://github.com/alesapin)). +* Backported in [#16310](https://github.com/ClickHouse/ClickHouse/issues/16310): Add allow_nondeterministic_optimize_skip_unused_shards (to allow non deterministic like rand() or dictGet() in sharding key). [#16105](https://github.com/ClickHouse/ClickHouse/pull/16105) ([Azat Khuzhin](https://github.com/azat)). + +#### Bug Fix +* Backported in [#15620](https://github.com/ClickHouse/ClickHouse/issues/15620): Throw an error when a single parameter is passed to ReplicatedMergeTree instead of ignoring it. [#15516](https://github.com/ClickHouse/ClickHouse/pull/15516) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#16203](https://github.com/ClickHouse/ClickHouse/issues/16203): Decrement the `ReadonlyReplica` metric when detaching read-only tables. This fixes [#15598](https://github.com/ClickHouse/ClickHouse/issues/15598). [#15592](https://github.com/ClickHouse/ClickHouse/pull/15592) ([sundyli](https://github.com/sundy-li)). +* Backported in [#16230](https://github.com/ClickHouse/ClickHouse/issues/16230): Fixed bug with globs in S3 table function, region from URL was not applied to S3 client configuration. [#15646](https://github.com/ClickHouse/ClickHouse/pull/15646) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#15871](https://github.com/ClickHouse/ClickHouse/issues/15871): Fix error `Cannot add simple transform to empty Pipe` which happened while reading from `Buffer` table which has different structure than destination table. It was possible if destination table returned empty result for query. Fixes [#15529](https://github.com/ClickHouse/ClickHouse/issues/15529). [#15662](https://github.com/ClickHouse/ClickHouse/pull/15662) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#15772](https://github.com/ClickHouse/ClickHouse/issues/15772): Fixed too low default value of `max_replicated_logs_to_keep` setting, which might cause replicas to become lost too often. Improve lost replica recovery process by choosing the most up-to-date replica to clone. Also do not remove old parts from lost replica, detach them instead. [#15701](https://github.com/ClickHouse/ClickHouse/pull/15701) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15738](https://github.com/ClickHouse/ClickHouse/issues/15738): Fix error `Cannot find column` which may happen at insertion into `MATERIALIZED VIEW` in case if query for `MV` containes `ARRAY JOIN`. [#15717](https://github.com/ClickHouse/ClickHouse/pull/15717) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#15795](https://github.com/ClickHouse/ClickHouse/issues/15795): Fix some cases of queries, in which only virtual columns are selected. Previously `Not found column _nothing in block` exception may be thrown. Fixes [#12298](https://github.com/ClickHouse/ClickHouse/issues/12298). [#15756](https://github.com/ClickHouse/ClickHouse/pull/15756) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#15900](https://github.com/ClickHouse/ClickHouse/issues/15900): Fix exception `Block structure mismatch` in `SELECT ... ORDER BY DESC` queries which were executed after `ALTER MODIFY COLUMN` query. Fixes [#15800](https://github.com/ClickHouse/ClickHouse/issues/15800). [#15852](https://github.com/ClickHouse/ClickHouse/pull/15852) ([alesapin](https://github.com/alesapin)). +* Backported in [#15925](https://github.com/ClickHouse/ClickHouse/issues/15925): Now exception will be thrown when `ALTER MODIFY COLUMN ... DEFAULT ...` has incompatible default with column type. Fixes [#15854](https://github.com/ClickHouse/ClickHouse/issues/15854). [#15858](https://github.com/ClickHouse/ClickHouse/pull/15858) ([alesapin](https://github.com/alesapin)). +* Backported in [#15920](https://github.com/ClickHouse/ClickHouse/issues/15920): Fix possible deadlocks in RBAC. [#15875](https://github.com/ClickHouse/ClickHouse/pull/15875) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#16168](https://github.com/ClickHouse/ClickHouse/issues/16168): Fix incorrect empty result for query from `Distributed` table if query has `WHERE`, `PREWHERE` and `GLOBAL IN`. Fixes [#15792](https://github.com/ClickHouse/ClickHouse/issues/15792). [#15933](https://github.com/ClickHouse/ClickHouse/pull/15933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#16355](https://github.com/ClickHouse/ClickHouse/issues/16355): Fixed `DROP TABLE IF EXISTS` failure with `Table ... doesn't exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) Fixed `DROP/DETACH DATABASE` failure with `Table ... doesn't exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15968](https://github.com/ClickHouse/ClickHouse/issues/15968): Fix a crash when database creation fails. [#15954](https://github.com/ClickHouse/ClickHouse/pull/15954) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#16025](https://github.com/ClickHouse/ClickHouse/issues/16025): Fix ambiguity in parsing of settings profiles: `CREATE USER ... SETTINGS profile readonly` is now considered as using a profile named `readonly`, not a setting named `profile` with the readonly constraint. This fixes [#15628](https://github.com/ClickHouse/ClickHouse/issues/15628). [#15982](https://github.com/ClickHouse/ClickHouse/pull/15982) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#16219](https://github.com/ClickHouse/ClickHouse/issues/16219): Fix rare segfaults when inserting into or selecting from MaterializedView and concurrently dropping target table (for Atomic database engine). [#15984](https://github.com/ClickHouse/ClickHouse/pull/15984) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#16029](https://github.com/ClickHouse/ClickHouse/issues/16029): Prevent replica hang for 5-10 mins when replication error happens after a period of inactivity. [#15987](https://github.com/ClickHouse/ClickHouse/pull/15987) ([filimonov](https://github.com/filimonov)). +* Backported in [#16088](https://github.com/ClickHouse/ClickHouse/issues/16088): Allow to use direct layout for dictionaries with complex keys. [#16007](https://github.com/ClickHouse/ClickHouse/pull/16007) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#16360](https://github.com/ClickHouse/ClickHouse/issues/16360): Fix collate name & charset name parser and support `length = 0` for string type. [#16008](https://github.com/ClickHouse/ClickHouse/pull/16008) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#16141](https://github.com/ClickHouse/ClickHouse/issues/16141): Fix `ALTER MODIFY ... ORDER BY` query hang for `ReplicatedVersionedCollapsingMergeTree`. This fixes [#15980](https://github.com/ClickHouse/ClickHouse/issues/15980). [#16011](https://github.com/ClickHouse/ClickHouse/pull/16011) ([alesapin](https://github.com/alesapin)). +* Backported in [#16078](https://github.com/ClickHouse/ClickHouse/issues/16078): Fixes [#15780](https://github.com/ClickHouse/ClickHouse/issues/15780) regression, e.g. indexOf([1, 2, 3], toLowCardinality(1)) now is prohibited but it should not be. [#16038](https://github.com/ClickHouse/ClickHouse/pull/16038) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#16294](https://github.com/ClickHouse/ClickHouse/issues/16294): Fix dictGet in sharding_key (and similar places, i.e. when the function context is stored permanently). [#16205](https://github.com/ClickHouse/ClickHouse/pull/16205) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16233](https://github.com/ClickHouse/ClickHouse/issues/16233): Fix the case when memory can be overallocated regardless to the limit. This closes [#14560](https://github.com/ClickHouse/ClickHouse/issues/14560). [#16206](https://github.com/ClickHouse/ClickHouse/pull/16206) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#16325](https://github.com/ClickHouse/ClickHouse/issues/16325): Fix a possible memory leak during `GROUP BY` with string keys, caused by an error in `TwoLevelStringHashTable` implementation. [#16264](https://github.com/ClickHouse/ClickHouse/pull/16264) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#16375](https://github.com/ClickHouse/ClickHouse/issues/16375): Fix async Distributed INSERT w/ prefer_localhost_replica=0 and internal_replication. [#16358](https://github.com/ClickHouse/ClickHouse/pull/16358) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16431](https://github.com/ClickHouse/ClickHouse/issues/16431): Fix group by with totals/rollup/cube modifers and min/max functions over group by keys. Fixes [#16393](https://github.com/ClickHouse/ClickHouse/issues/16393). [#16397](https://github.com/ClickHouse/ClickHouse/pull/16397) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#16450](https://github.com/ClickHouse/ClickHouse/issues/16450): Fix double free in case of exception in function `dictGet`. It could have happened if dictionary was loaded with error. [#16429](https://github.com/ClickHouse/ClickHouse/pull/16429) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Other +* Unfold `{database}`, `{table}` and `{uuid}` macros in `ReplicatedMergeTree` arguments on table creation. [#16159](https://github.com/ClickHouse/ClickHouse/pull/16159) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v20.8.6.6-lts.md b/docs/changelogs/v20.8.6.6-lts.md new file mode 100644 index 00000000000..d63266df523 --- /dev/null +++ b/docs/changelogs/v20.8.6.6-lts.md @@ -0,0 +1,12 @@ +### ClickHouse release v20.8.6.6-lts FIXME as compared to v20.8.5.45-lts + +#### Bug Fix +* Backported in [#16494](https://github.com/ClickHouse/ClickHouse/issues/16494): Fix bug with MySQL database. When MySQL server used as database engine is down some queries raise Exception, because they try to get tables from disabled server, while it's unnecessary. For example, query `SELECT ... FROM system.parts` should work only with MergeTree tables and don't touch MySQL database at all. [#16032](https://github.com/ClickHouse/ClickHouse/pull/16032) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#16816](https://github.com/ClickHouse/ClickHouse/issues/16816): Fixed the inconsistent behaviour when a part of return data could be dropped because the set for its filtration wasn't created. [#16308](https://github.com/ClickHouse/ClickHouse/pull/16308) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#16506](https://github.com/ClickHouse/ClickHouse/issues/16506): Fix processing of very large entries in replication queue. Very large entries may appear in ALTER queries if table structure is extremely large (near 1 MB). This fixes [#16307](https://github.com/ClickHouse/ClickHouse/issues/16307). [#16332](https://github.com/ClickHouse/ClickHouse/pull/16332) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#16472](https://github.com/ClickHouse/ClickHouse/issues/16472): Fix DROP TABLE for Distributed (racy with INSERT). [#16409](https://github.com/ClickHouse/ClickHouse/pull/16409) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16573](https://github.com/ClickHouse/ClickHouse/issues/16573): Fix rapid growth of metadata when using MySQL Master -> MySQL Slave -> ClickHouse MaterializeMySQL Engine, and `slave_parallel_worker` enabled on MySQL Slave, by properly shrinking GTID sets. This fixes [#15951](https://github.com/ClickHouse/ClickHouse/issues/15951). [#16504](https://github.com/ClickHouse/ClickHouse/pull/16504) ([TCeason](https://github.com/TCeason)). +* Backported in [#16552](https://github.com/ClickHouse/ClickHouse/issues/16552): Now when parsing AVRO from input the LowCardinality is removed from type. Fixes [#16188](https://github.com/ClickHouse/ClickHouse/issues/16188). [#16521](https://github.com/ClickHouse/ClickHouse/pull/16521) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#16823](https://github.com/ClickHouse/ClickHouse/issues/16823): Fixed [#16081](https://github.com/ClickHouse/ClickHouse/issues/16081). [#16613](https://github.com/ClickHouse/ClickHouse/pull/16613) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#16892](https://github.com/ClickHouse/ClickHouse/issues/16892): Fix rare silent crashes when query profiler is on and ClickHouse is installed on OS with glibc version that has (supposedly) broken asynchronous unwind tables for some functions. This fixes [#15301](https://github.com/ClickHouse/ClickHouse/issues/15301). This fixes [#13098](https://github.com/ClickHouse/ClickHouse/issues/13098). [#16846](https://github.com/ClickHouse/ClickHouse/pull/16846) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.8.7.15-lts.md b/docs/changelogs/v20.8.7.15-lts.md new file mode 100644 index 00000000000..826e4f7e070 --- /dev/null +++ b/docs/changelogs/v20.8.7.15-lts.md @@ -0,0 +1,19 @@ +### ClickHouse release v20.8.7.15-lts FIXME as compared to v20.8.6.6-lts + +#### Improvement +* Backported in [#17029](https://github.com/ClickHouse/ClickHouse/issues/17029): Make it possible to connect to `clickhouse-server` secure endpoint which requires SNI. This is possible when `clickhouse-server` is hosted behind TLS proxy. [#16938](https://github.com/ClickHouse/ClickHouse/pull/16938) ([filimonov](https://github.com/filimonov)). + +#### Bug Fix +* Backported in [#15676](https://github.com/ClickHouse/ClickHouse/issues/15676): Query is finished faster in case of exception. Cancel execution on remote replicas if exception happens. [#15578](https://github.com/ClickHouse/ClickHouse/pull/15578) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#17106](https://github.com/ClickHouse/ClickHouse/issues/17106): fixes [#16574](https://github.com/ClickHouse/ClickHouse/issues/16574) fixes [#16231](https://github.com/ClickHouse/ClickHouse/issues/16231) fix remote query failure when using 'if' suffix aggregate function. [#16610](https://github.com/ClickHouse/ClickHouse/pull/16610) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#16761](https://github.com/ClickHouse/ClickHouse/issues/16761): This will fix optimize_read_in_order/optimize_aggregation_in_order with max_threads>0 and expression in ORDER BY. [#16637](https://github.com/ClickHouse/ClickHouse/pull/16637) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16738](https://github.com/ClickHouse/ClickHouse/issues/16738): Fix `IN` operator over several columns and tuples with enabled `transform_null_in` setting. Fixes [#15310](https://github.com/ClickHouse/ClickHouse/issues/15310). [#16722](https://github.com/ClickHouse/ClickHouse/pull/16722) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17021](https://github.com/ClickHouse/ClickHouse/issues/17021): Fix crash when using `any` without any arguments. This is for [#16803](https://github.com/ClickHouse/ClickHouse/issues/16803) . cc @azat. [#16826](https://github.com/ClickHouse/ClickHouse/pull/16826) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#16880](https://github.com/ClickHouse/ClickHouse/issues/16880): Abort multipart upload if no data was written to WriteBufferFromS3. [#16840](https://github.com/ClickHouse/ClickHouse/pull/16840) ([Pavel Kovalenko](https://github.com/Jokser)). +* Backported in [#16950](https://github.com/ClickHouse/ClickHouse/issues/16950): Prevent clickhouse server crashes when using TimeSeriesGroupSum. [#16865](https://github.com/ClickHouse/ClickHouse/pull/16865) ([filimonov](https://github.com/filimonov)). +* Backported in [#17077](https://github.com/ClickHouse/ClickHouse/issues/17077): Fix possible error `Illegal type of argument` for queries with `ORDER BY`. Fixes [#16580](https://github.com/ClickHouse/ClickHouse/issues/16580). [#16928](https://github.com/ClickHouse/ClickHouse/pull/16928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17007](https://github.com/ClickHouse/ClickHouse/issues/17007): Install script should always create subdirs in config folders. This is only relevant for Docker build with custom config. [#16936](https://github.com/ClickHouse/ClickHouse/pull/16936) ([filimonov](https://github.com/filimonov)). +* Backported in [#17011](https://github.com/ClickHouse/ClickHouse/issues/17011): Fix possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17040](https://github.com/ClickHouse/ClickHouse/issues/17040): Reresolve the IP of the `format_avro_schema_registry_url` in case of errors. [#16985](https://github.com/ClickHouse/ClickHouse/pull/16985) ([filimonov](https://github.com/filimonov)). +* Backported in [#17171](https://github.com/ClickHouse/ClickHouse/issues/17171): Fix bug when `ON CLUSTER` queries may hang forever for non-leader ReplicatedMergeTreeTables. [#17089](https://github.com/ClickHouse/ClickHouse/pull/17089) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v20.8.8.2-lts.md b/docs/changelogs/v20.8.8.2-lts.md new file mode 100644 index 00000000000..73e566efac2 --- /dev/null +++ b/docs/changelogs/v20.8.8.2-lts.md @@ -0,0 +1,10 @@ +### ClickHouse release v20.8.8.2-lts FIXME as compared to v20.8.7.15-lts + +#### Bug Fix +* Backported in [#17198](https://github.com/ClickHouse/ClickHouse/issues/17198): Avoid unnecessary network errors for remote queries which may be cancelled while execution, like queries with `LIMIT`. [#17006](https://github.com/ClickHouse/ClickHouse/pull/17006) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#17131](https://github.com/ClickHouse/ClickHouse/issues/17131): Fixed crash on `CREATE TABLE ... AS some_table` query when `some_table` was created `AS table_function()` Fixes [#16944](https://github.com/ClickHouse/ClickHouse/issues/16944). [#17072](https://github.com/ClickHouse/ClickHouse/pull/17072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17192](https://github.com/ClickHouse/ClickHouse/issues/17192): Fix ColumnConst comparison which leads to crash. This fixed [#17088](https://github.com/ClickHouse/ClickHouse/issues/17088) . [#17135](https://github.com/ClickHouse/ClickHouse/pull/17135) ([Amos Bird](https://github.com/amosbird)). + +#### Build/Testing/Packaging Improvement +* Backported in [#17287](https://github.com/ClickHouse/ClickHouse/issues/17287): Update embedded timezone data to version 2020d (also update cctz to the latest master). [#17204](https://github.com/ClickHouse/ClickHouse/pull/17204) ([filimonov](https://github.com/filimonov)). + diff --git a/docs/changelogs/v20.8.9.6-lts.md b/docs/changelogs/v20.8.9.6-lts.md new file mode 100644 index 00000000000..943bd9e57e7 --- /dev/null +++ b/docs/changelogs/v20.8.9.6-lts.md @@ -0,0 +1,2 @@ +### ClickHouse release v20.8.9.6-lts FIXME as compared to v20.8.8.2-lts + diff --git a/docs/changelogs/v20.9.1.4585-prestable.md b/docs/changelogs/v20.9.1.4585-prestable.md new file mode 100644 index 00000000000..db3b8a409a4 --- /dev/null +++ b/docs/changelogs/v20.9.1.4585-prestable.md @@ -0,0 +1,63 @@ +### ClickHouse release v20.9.1.4585-prestable FIXME as compared to v20.8.1.4513-prestable + +#### Backward Incompatible Change +* Added MergeTree settings (`max_replicated_merges_with_ttl_in_queue` and `max_number_of_merges_with_ttl_in_pool`) to control the number of merges with TTL in the background pool and replicated queue. This change breaks compatibility with older versions only if you use delete TTL. Otherwise, replication will stay compatible. You can avoid incompatibility issues if you update all shard replicas at once or execute `SYSTEM STOP TTL MERGES` until you finish the update of all replicas. If you'll get an incompatible entry in the replication queue, first of all, execute `SYSTEM STOP TTL MERGES` and after `ALTER TABLE ... DETACH PARTITION ...` the partition where incompatible TTL merge was assigned. Attach it back on a single replica. [#14490](https://github.com/ClickHouse/ClickHouse/pull/14490) ([alesapin](https://github.com/alesapin)). + +#### New Feature +* Add table function `view` which turns an subquery into a table object. This helps passing queries around. For instance, it can be used in remote/cluster table functions. [#12567](https://github.com/ClickHouse/ClickHouse/pull/12567) ([Amos Bird](https://github.com/amosbird)). +* Now we can write `select * apply(length) apply(max) from wide_string_table` to find out the maxium length of all string columns. And the follow two variants are provided too:. [#14233](https://github.com/ClickHouse/ClickHouse/pull/14233) ([Amos Bird](https://github.com/amosbird)). +* Add `query_start_time_microseconds` field to `system.query_log` & `system.query_thread_log` tables. [#14252](https://github.com/ClickHouse/ClickHouse/pull/14252) ([Bharat Nallan](https://github.com/bharatnc)). +* Added an aggregate function RankCorrelationSpearman which simply computes a rank correlation coefficient. Continuation of [#11769](https://github.com/ClickHouse/ClickHouse/issues/11769). [#14411](https://github.com/ClickHouse/ClickHouse/pull/14411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added database generation by query util. Continuation of [#10973](https://github.com/ClickHouse/ClickHouse/issues/10973). [#14442](https://github.com/ClickHouse/ClickHouse/pull/14442) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### Performance Improvement +* Optimize queries with LIMIT/LIMIT BY/ORDER BY for distributed with GROUP BY sharding_key (under optimize_skip_unused_shards and optimize_distributed_group_by_sharding_key). [#10373](https://github.com/ClickHouse/ClickHouse/pull/10373) ([Azat Khuzhin](https://github.com/azat)). + +#### Improvement +* Improvements in StorageRabbitMQ: Added connection and channels failure handling, proper commits, insert failures handling, better exchanges, queue durability and queue resume opportunity, new queue settings. Fixed tests. [#12761](https://github.com/ClickHouse/ClickHouse/pull/12761) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Added Redis requirepass authorization. [#13688](https://github.com/ClickHouse/ClickHouse/pull/13688) ([Ivan Torgashov](https://github.com/it1804)). +* Add precision argument for DateTime type. [#13761](https://github.com/ClickHouse/ClickHouse/pull/13761) ([Winter Zhang](https://github.com/zhang2014)). +* Improve the Kafka engine performance by providing independent thread for each consumer. Separate thread pool for streaming engines (like Kafka). [#13939](https://github.com/ClickHouse/ClickHouse/pull/13939) ([fastio](https://github.com/fastio)). +* Add default compression codec for parts in `system.part_log` with name `default_compression_codec`. [#14116](https://github.com/ClickHouse/ClickHouse/pull/14116) ([alesapin](https://github.com/alesapin)). +* Replace wide integers from boost multiprecision with implementation from https://github.com/cerevra/int. [#14229](https://github.com/ClickHouse/ClickHouse/pull/14229) ([Artem Zuikov](https://github.com/4ertus2)). +* Implicitly convert primary key to not null in MaterializeMySQL(Same as MySQL). Fixes [#14114](https://github.com/ClickHouse/ClickHouse/issues/14114). [#14397](https://github.com/ClickHouse/ClickHouse/pull/14397) ([Winter Zhang](https://github.com/zhang2014)). +* Added new setting system_events_show_zero_values as proposed in [#11384](https://github.com/ClickHouse/ClickHouse/issues/11384). [#14404](https://github.com/ClickHouse/ClickHouse/pull/14404) ([Dmitry Rubashkin](https://github.com/dimarub2000)). +* Now obfuscator supports UUID type as proposed in [#13163](https://github.com/ClickHouse/ClickHouse/issues/13163). [#14409](https://github.com/ClickHouse/ClickHouse/pull/14409) ([Dmitry Rubashkin](https://github.com/dimarub2000)). +* Creating sets for multiple `JOIN` and `IN` in parallel. It may slightly improve performance for queries with several different `IN subquery` expressions. [#14412](https://github.com/ClickHouse/ClickHouse/pull/14412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Now TTLs will be applied during merge if they were not previously materialized. [#14438](https://github.com/ClickHouse/ClickHouse/pull/14438) ([alesapin](https://github.com/alesapin)). +* MySQL handler returns `OK` for queries like `SET @@var = value`. Such statement is ignored. It is needed because some MySQL drivers send `SET @@` query for setup after handshake [#9336](https://github.com/ClickHouse/ClickHouse/issues/9336)#issuecomment-686222422 . [#14469](https://github.com/ClickHouse/ClickHouse/pull/14469) ([BohuTANG](https://github.com/BohuTANG)). +* Speed up server shutdown process if there are ongoing S3 requests. [#14496](https://github.com/ClickHouse/ClickHouse/pull/14496) ([Pavel Kovalenko](https://github.com/Jokser)). +* Disallow empty time_zone argument in `toStartOf*` type of functions. [#14509](https://github.com/ClickHouse/ClickHouse/pull/14509) ([Bharat Nallan](https://github.com/bharatnc)). +* ... [#14523](https://github.com/ClickHouse/ClickHouse/pull/14523) ([BohuTANG](https://github.com/BohuTANG)). +* Use std::filesystem::path in ConfigProcessor for concatenating file paths. [#14558](https://github.com/ClickHouse/ClickHouse/pull/14558) ([Bharat Nallan](https://github.com/bharatnc)). + +#### Bug Fix +* Fix arrayJoin() capturing in lambda (LOGICAL_ERROR). [#13792](https://github.com/ClickHouse/ClickHouse/pull/13792) ([Azat Khuzhin](https://github.com/azat)). +* Fix GRANT ALL statement when executed on a non-global level. [#13987](https://github.com/ClickHouse/ClickHouse/pull/13987) ([Vitaly Baranov](https://github.com/vitlibar)). +* Disallows `CODEC` on `ALIAS` column type. Fixes [#13911](https://github.com/ClickHouse/ClickHouse/issues/13911). [#14263](https://github.com/ClickHouse/ClickHouse/pull/14263) ([Bharat Nallan](https://github.com/bharatnc)). +* Better check for tuple size in SSD cache complex key external dictionaries. This fixes [#13981](https://github.com/ClickHouse/ClickHouse/issues/13981). [#14313](https://github.com/ClickHouse/ClickHouse/pull/14313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix QueryPlan lifetime (for EXPLAIN PIPELINE graph=1) for queries with nested interpreter. [#14315](https://github.com/ClickHouse/ClickHouse/pull/14315) ([Azat Khuzhin](https://github.com/azat)). +* Fix exception during ALTER LIVE VIEW query with REFRESH command. [#14320](https://github.com/ClickHouse/ClickHouse/pull/14320) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix crash during `ALTER` query for table which was created `AS table_function`. Fixes [#14212](https://github.com/ClickHouse/ClickHouse/issues/14212). [#14326](https://github.com/ClickHouse/ClickHouse/pull/14326) ([alesapin](https://github.com/alesapin)). +* Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. [#14334](https://github.com/ClickHouse/ClickHouse/pull/14334) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. Continuation of [#14334](https://github.com/ClickHouse/ClickHouse/issues/14334). [#14402](https://github.com/ClickHouse/ClickHouse/pull/14402) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix bug which leads to wrong merges assignment if table has partitions with a single part. [#14444](https://github.com/ClickHouse/ClickHouse/pull/14444) ([alesapin](https://github.com/alesapin)). +* Proxy restart/start/stop/reload of SysVinit to systemd (if it is used). [#14460](https://github.com/ClickHouse/ClickHouse/pull/14460) ([Azat Khuzhin](https://github.com/azat)). +* Check for array size overflow in `topK` aggregate function. Without this check the user may send a query with carefully crafter parameters that will lead to server crash. This closes [#14452](https://github.com/ClickHouse/ClickHouse/issues/14452). [#14467](https://github.com/ClickHouse/ClickHouse/pull/14467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Build/Testing/Packaging Improvement +* Integration tests use default base config. All config changes are explicit with main_configs, user_configs and dictionaries parameters for instance. [#13647](https://github.com/ClickHouse/ClickHouse/pull/13647) ([Ilya Yatsishin](https://github.com/qoega)). +* ... [#14368](https://github.com/ClickHouse/ClickHouse/pull/14368) ([BohuTANG](https://github.com/BohuTANG)). +* Fix the logic in backport script. In previous versions it was triggered for any labels of 100% red color. It was strange. [#14433](https://github.com/ClickHouse/ClickHouse/pull/14433) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix missed `#include `. [#14440](https://github.com/ClickHouse/ClickHouse/pull/14440) ([Matwey V. Kornilov](https://github.com/matwey)). +* Prepare for build with clang 11. [#14455](https://github.com/ClickHouse/ClickHouse/pull/14455) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Lower binary size in debug build by removing debug info from `Functions`. This is needed only for one internal project in Yandex who is using very old linker. [#14549](https://github.com/ClickHouse/ClickHouse/pull/14549) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable ccache by default in cmake if it's found in OS. [#14575](https://github.com/ClickHouse/ClickHouse/pull/14575) ([alesapin](https://github.com/alesapin)). + +#### Other +* Changelog for 20.7 [#13499](https://github.com/ClickHouse/ClickHouse/issues/13499). [#14420](https://github.com/ClickHouse/ClickHouse/pull/14420) ([Alexander Kazakov](https://github.com/Akazz)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Less number of threads in builder"'. [#14421](https://github.com/ClickHouse/ClickHouse/pull/14421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.9.2.20-stable.md b/docs/changelogs/v20.9.2.20-stable.md new file mode 100644 index 00000000000..6c869e47a8f --- /dev/null +++ b/docs/changelogs/v20.9.2.20-stable.md @@ -0,0 +1,75 @@ +### ClickHouse release v20.9.2.20-stable FIXME as compared to v20.8.1.4513-prestable + +#### New Feature +* Add table function `view` which turns an subquery into a table object. This helps passing queries around. For instance, it can be used in remote/cluster table functions. [#12567](https://github.com/ClickHouse/ClickHouse/pull/12567) ([Amos Bird](https://github.com/amosbird)). +* Now we can write `select * apply(length) apply(max) from wide_string_table` to find out the maxium length of all string columns. And the follow two variants are provided too:. [#14233](https://github.com/ClickHouse/ClickHouse/pull/14233) ([Amos Bird](https://github.com/amosbird)). +* Added an aggregate function RankCorrelationSpearman which simply computes a rank correlation coefficient. Continuation of [#11769](https://github.com/ClickHouse/ClickHouse/issues/11769). [#14411](https://github.com/ClickHouse/ClickHouse/pull/14411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added database generation by query util. Continuation of [#10973](https://github.com/ClickHouse/ClickHouse/issues/10973). [#14442](https://github.com/ClickHouse/ClickHouse/pull/14442) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### Performance Improvement +* Optimize queries with LIMIT/LIMIT BY/ORDER BY for distributed with GROUP BY sharding_key (under optimize_skip_unused_shards and optimize_distributed_group_by_sharding_key). [#10373](https://github.com/ClickHouse/ClickHouse/pull/10373) ([Azat Khuzhin](https://github.com/azat)). + +#### Improvement +* Improvements in StorageRabbitMQ: Added connection and channels failure handling, proper commits, insert failures handling, better exchanges, queue durability and queue resume opportunity, new queue settings. Fixed tests. [#12761](https://github.com/ClickHouse/ClickHouse/pull/12761) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Added Redis requirepass authorization. [#13688](https://github.com/ClickHouse/ClickHouse/pull/13688) ([Ivan Torgashov](https://github.com/it1804)). +* Add precision argument for DateTime type. [#13761](https://github.com/ClickHouse/ClickHouse/pull/13761) ([Winter Zhang](https://github.com/zhang2014)). +* Improve the Kafka engine performance by providing independent thread for each consumer. Separate thread pool for streaming engines (like Kafka). [#13939](https://github.com/ClickHouse/ClickHouse/pull/13939) ([fastio](https://github.com/fastio)). +* Add default compression codec for parts in `system.part_log` with name `default_compression_codec`. [#14116](https://github.com/ClickHouse/ClickHouse/pull/14116) ([alesapin](https://github.com/alesapin)). +* Replace wide integers from boost multiprecision with implementation from https://github.com/cerevra/int. [#14229](https://github.com/ClickHouse/ClickHouse/pull/14229) ([Artem Zuikov](https://github.com/4ertus2)). +* Implicitly convert primary key to not null in MaterializeMySQL(Same as MySQL). Fixes [#14114](https://github.com/ClickHouse/ClickHouse/issues/14114). [#14397](https://github.com/ClickHouse/ClickHouse/pull/14397) ([Winter Zhang](https://github.com/zhang2014)). +* Added new setting system_events_show_zero_values as proposed in [#11384](https://github.com/ClickHouse/ClickHouse/issues/11384). [#14404](https://github.com/ClickHouse/ClickHouse/pull/14404) ([Dmitry Rubashkin](https://github.com/dimarub2000)). +* Now obfuscator supports UUID type as proposed in [#13163](https://github.com/ClickHouse/ClickHouse/issues/13163). [#14409](https://github.com/ClickHouse/ClickHouse/pull/14409) ([Dmitry Rubashkin](https://github.com/dimarub2000)). +* Creating sets for multiple `JOIN` and `IN` in parallel. It may slightly improve performance for queries with several different `IN subquery` expressions. [#14412](https://github.com/ClickHouse/ClickHouse/pull/14412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Now TTLs will be applied during merge if they were not previously materialized. [#14438](https://github.com/ClickHouse/ClickHouse/pull/14438) ([alesapin](https://github.com/alesapin)). +* MySQL handler returns `OK` for queries like `SET @@var = value`. Such statement is ignored. It is needed because some MySQL drivers send `SET @@` query for setup after handshake [#9336](https://github.com/ClickHouse/ClickHouse/issues/9336)#issuecomment-686222422 . [#14469](https://github.com/ClickHouse/ClickHouse/pull/14469) ([BohuTANG](https://github.com/BohuTANG)). +* Disallow empty time_zone argument in `toStartOf*` type of functions. [#14509](https://github.com/ClickHouse/ClickHouse/pull/14509) ([Bharat Nallan](https://github.com/bharatnc)). +* ... [#14523](https://github.com/ClickHouse/ClickHouse/pull/14523) ([BohuTANG](https://github.com/BohuTANG)). +* Backported in [#14873](https://github.com/ClickHouse/ClickHouse/issues/14873): Allow using multi-volume storage configuration in storage Distributed. [#14839](https://github.com/ClickHouse/ClickHouse/pull/14839) ([Pavel Kovalenko](https://github.com/Jokser)). + +#### Bug Fix +* Fix arrayJoin() capturing in lambda (LOGICAL_ERROR). [#13792](https://github.com/ClickHouse/ClickHouse/pull/13792) ([Azat Khuzhin](https://github.com/azat)). +* Fix GRANT ALL statement when executed on a non-global level. [#13987](https://github.com/ClickHouse/ClickHouse/pull/13987) ([Vitaly Baranov](https://github.com/vitlibar)). +* Disallows `CODEC` on `ALIAS` column type. Fixes [#13911](https://github.com/ClickHouse/ClickHouse/issues/13911). [#14263](https://github.com/ClickHouse/ClickHouse/pull/14263) ([Bharat Nallan](https://github.com/bharatnc)). +* Better check for tuple size in SSD cache complex key external dictionaries. This fixes [#13981](https://github.com/ClickHouse/ClickHouse/issues/13981). [#14313](https://github.com/ClickHouse/ClickHouse/pull/14313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix QueryPlan lifetime (for EXPLAIN PIPELINE graph=1) for queries with nested interpreter. [#14315](https://github.com/ClickHouse/ClickHouse/pull/14315) ([Azat Khuzhin](https://github.com/azat)). +* Fix exception during ALTER LIVE VIEW query with REFRESH command. [#14320](https://github.com/ClickHouse/ClickHouse/pull/14320) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix crash during `ALTER` query for table which was created `AS table_function`. Fixes [#14212](https://github.com/ClickHouse/ClickHouse/issues/14212). [#14326](https://github.com/ClickHouse/ClickHouse/pull/14326) ([alesapin](https://github.com/alesapin)). +* Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. [#14334](https://github.com/ClickHouse/ClickHouse/pull/14334) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Stop query execution if exception happened in `PipelineExecutor` itself. This could prevent rare possible query hung. Continuation of [#14334](https://github.com/ClickHouse/ClickHouse/issues/14334). [#14402](https://github.com/ClickHouse/ClickHouse/pull/14402) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix bug which leads to wrong merges assignment if table has partitions with a single part. [#14444](https://github.com/ClickHouse/ClickHouse/pull/14444) ([alesapin](https://github.com/alesapin)). +* Proxy restart/start/stop/reload of SysVinit to systemd (if it is used). [#14460](https://github.com/ClickHouse/ClickHouse/pull/14460) ([Azat Khuzhin](https://github.com/azat)). +* Check for array size overflow in `topK` aggregate function. Without this check the user may send a query with carefully crafter parameters that will lead to server crash. This closes [#14452](https://github.com/ClickHouse/ClickHouse/issues/14452). [#14467](https://github.com/ClickHouse/ClickHouse/pull/14467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15020](https://github.com/ClickHouse/ClickHouse/issues/15020): Fixed the incorrect sorting order of `Nullable` column. This fixes [#14344](https://github.com/ClickHouse/ClickHouse/issues/14344). [#14495](https://github.com/ClickHouse/ClickHouse/pull/14495) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14599](https://github.com/ClickHouse/ClickHouse/issues/14599): Fix rare segfaults in functions with combinator -Resample, which could appear in result of overflow with very large parameters. [#14562](https://github.com/ClickHouse/ClickHouse/pull/14562) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#14729](https://github.com/ClickHouse/ClickHouse/issues/14729): Cleanup data directory after Zookeeper exceptions during CreateQuery for StorageReplicatedMergeTree Engine. [#14563](https://github.com/ClickHouse/ClickHouse/pull/14563) ([Bharat Nallan](https://github.com/bharatnc)). +* Backported in [#14654](https://github.com/ClickHouse/ClickHouse/issues/14654): Added the checker as neither calling `lc->isNullable()` nor calling `ls->getDictionaryPtr()->isNullable()` would return the correct result. [#14591](https://github.com/ClickHouse/ClickHouse/pull/14591) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#14663](https://github.com/ClickHouse/ClickHouse/issues/14663): Fix wrong Decimal multiplication result caused wrong decimal scale of result column. [#14603](https://github.com/ClickHouse/ClickHouse/pull/14603) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#14662](https://github.com/ClickHouse/ClickHouse/issues/14662): Stuff the query into ASTFunction's argument list so that we don't break the presumptions of some AST visitors. This fixes [#14608](https://github.com/ClickHouse/ClickHouse/issues/14608). [#14611](https://github.com/ClickHouse/ClickHouse/pull/14611) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#14792](https://github.com/ClickHouse/ClickHouse/issues/14792): Fix bug when `ALTER UPDATE` mutation with Nullable column in assignment expression and constant value (like `UPDATE x = 42`) leads to incorrect value in column or segfault. Fixes [#13634](https://github.com/ClickHouse/ClickHouse/issues/13634), [#14045](https://github.com/ClickHouse/ClickHouse/issues/14045). [#14646](https://github.com/ClickHouse/ClickHouse/pull/14646) ([alesapin](https://github.com/alesapin)). +* Backported in [#14721](https://github.com/ClickHouse/ClickHouse/issues/14721): Fixed missed default database name in metadata of materialized view when executing `ALTER ... MODIFY QUERY`. [#14664](https://github.com/ClickHouse/ClickHouse/pull/14664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#14770](https://github.com/ClickHouse/ClickHouse/issues/14770): Replace column transformer should replace identifiers with cloned ASTs. This fixes [#14695](https://github.com/ClickHouse/ClickHouse/issues/14695) . [#14734](https://github.com/ClickHouse/ClickHouse/pull/14734) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#14825](https://github.com/ClickHouse/ClickHouse/issues/14825): Fix wrong monotonicity detection for shrunk `Int -> Int` cast of signed types. It might lead to incorrect query result. This bug is unveiled in [#14513](https://github.com/ClickHouse/ClickHouse/issues/14513). [#14783](https://github.com/ClickHouse/ClickHouse/pull/14783) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#14912](https://github.com/ClickHouse/ClickHouse/issues/14912): Fix SIGSEGV for an attempt to INSERT into StorageFile(fd). [#14887](https://github.com/ClickHouse/ClickHouse/pull/14887) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#14944](https://github.com/ClickHouse/ClickHouse/issues/14944): Fix the issue when some invocations of `extractAllGroups` function may trigger "Memory limit exceeded" error. This fixes [#13383](https://github.com/ClickHouse/ClickHouse/issues/13383). [#14889](https://github.com/ClickHouse/ClickHouse/pull/14889) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#14956](https://github.com/ClickHouse/ClickHouse/issues/14956): Fixed `.metadata.tmp File exists` error when using `MaterializeMySQL` database engine. [#14898](https://github.com/ClickHouse/ClickHouse/pull/14898) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#14989](https://github.com/ClickHouse/ClickHouse/issues/14989): Publish CPU frequencies per logical core in `system.asynchronous_metrics`. This fixes [#14923](https://github.com/ClickHouse/ClickHouse/issues/14923). [#14924](https://github.com/ClickHouse/ClickHouse/pull/14924) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#15055](https://github.com/ClickHouse/ClickHouse/issues/15055): Now settings `number_of_free_entries_in_pool_to_execute_mutation` and `number_of_free_entries_in_pool_to_lower_max_size_of_merge` can be equal to `background_pool_size`. [#14975](https://github.com/ClickHouse/ClickHouse/pull/14975) ([alesapin](https://github.com/alesapin)). +* Backported in [#15079](https://github.com/ClickHouse/ClickHouse/issues/15079): Fix crash in RIGHT or FULL JOIN with join_algorith='auto' when memory limit exceeded and we should change HashJoin with MergeJoin. [#15002](https://github.com/ClickHouse/ClickHouse/pull/15002) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15059](https://github.com/ClickHouse/ClickHouse/issues/15059): If function `bar` was called with specifically crafted arguments, buffer overflow was possible. This closes [#13926](https://github.com/ClickHouse/ClickHouse/issues/13926). [#15028](https://github.com/ClickHouse/ClickHouse/pull/15028) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15052](https://github.com/ClickHouse/ClickHouse/issues/15052): We already use padded comparison between String and FixedString (https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionsComparison.h#L333). This PR applies the same logic to field comparison which corrects the usage of FixedString as primary keys. This fixes [#14908](https://github.com/ClickHouse/ClickHouse/issues/14908). [#15033](https://github.com/ClickHouse/ClickHouse/pull/15033) ([Amos Bird](https://github.com/amosbird)). + +#### Build/Testing/Packaging Improvement +* Integration tests use default base config. All config changes are explicit with main_configs, user_configs and dictionaries parameters for instance. [#13647](https://github.com/ClickHouse/ClickHouse/pull/13647) ([Ilya Yatsishin](https://github.com/qoega)). +* ... [#14368](https://github.com/ClickHouse/ClickHouse/pull/14368) ([BohuTANG](https://github.com/BohuTANG)). +* Fix the logic in backport script. In previous versions it was triggered for any labels of 100% red color. It was strange. [#14433](https://github.com/ClickHouse/ClickHouse/pull/14433) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix missed `#include `. [#14440](https://github.com/ClickHouse/ClickHouse/pull/14440) ([Matwey V. Kornilov](https://github.com/matwey)). +* Prepare for build with clang 11. [#14455](https://github.com/ClickHouse/ClickHouse/pull/14455) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Lower binary size in debug build by removing debug info from `Functions`. This is needed only for one internal project in Yandex who is using very old linker. [#14549](https://github.com/ClickHouse/ClickHouse/pull/14549) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Other +* Changelog for 20.7 [#13499](https://github.com/ClickHouse/ClickHouse/issues/13499). [#14420](https://github.com/ClickHouse/ClickHouse/pull/14420) ([Alexander Kazakov](https://github.com/Akazz)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Less number of threads in builder"'. [#14421](https://github.com/ClickHouse/ClickHouse/pull/14421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.9.3.45-stable.md b/docs/changelogs/v20.9.3.45-stable.md new file mode 100644 index 00000000000..49cebd73525 --- /dev/null +++ b/docs/changelogs/v20.9.3.45-stable.md @@ -0,0 +1,33 @@ +### ClickHouse release v20.9.3.45-stable FIXME as compared to v20.9.2.20-stable + +#### Improvement +* Backported in [#15568](https://github.com/ClickHouse/ClickHouse/issues/15568): Now it's possible to change the type of version column for `VersionedCollapsingMergeTree` with `ALTER` query. [#15442](https://github.com/ClickHouse/ClickHouse/pull/15442) ([alesapin](https://github.com/alesapin)). + +#### Bug Fix +* Backported in [#15150](https://github.com/ClickHouse/ClickHouse/issues/15150): Fix a problem where the server may get stuck on startup while talking to ZooKeeper, if the configuration files have to be fetched from ZK (using the `from_zk` include option). This fixes [#14814](https://github.com/ClickHouse/ClickHouse/issues/14814). [#14843](https://github.com/ClickHouse/ClickHouse/pull/14843) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#15250](https://github.com/ClickHouse/ClickHouse/issues/15250): Fixed segfault in CacheDictionary [#14837](https://github.com/ClickHouse/ClickHouse/issues/14837). [#14879](https://github.com/ClickHouse/ClickHouse/pull/14879) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#14971](https://github.com/ClickHouse/ClickHouse/issues/14971): Fix to make predicate push down work when subquery contains finalizeAggregation function. Fixes [#14847](https://github.com/ClickHouse/ClickHouse/issues/14847). [#14937](https://github.com/ClickHouse/ClickHouse/pull/14937) ([filimonov](https://github.com/filimonov)). +* Backported in [#15104](https://github.com/ClickHouse/ClickHouse/issues/15104): Fixed `Cannot rename ... errno: 22, strerror: Invalid argument` error on DDL query execution in Atomic database when running clickhouse-server in docker on Mac OS. [#15024](https://github.com/ClickHouse/ClickHouse/pull/15024) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15141](https://github.com/ClickHouse/ClickHouse/issues/15141): Fixes `Data compressed with different methods` in `join_algorithm='auto'`. Keep LowCardinality as type for left table join key in `join_algorithm='partial_merge'`. [#15088](https://github.com/ClickHouse/ClickHouse/pull/15088) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15209](https://github.com/ClickHouse/ClickHouse/issues/15209): Adjust decimals field size in mysql column definition packet. [#15152](https://github.com/ClickHouse/ClickHouse/pull/15152) ([maqroll](https://github.com/maqroll)). +* Backported in [#15224](https://github.com/ClickHouse/ClickHouse/issues/15224): Fix bug in table engine `Buffer` which doesn't allow to insert data of new structure into `Buffer` after `ALTER` query. Fixes [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117). [#15192](https://github.com/ClickHouse/ClickHouse/pull/15192) ([alesapin](https://github.com/alesapin)). +* Backported in [#15403](https://github.com/ClickHouse/ClickHouse/issues/15403): Fix instance crash when using joinGet with LowCardinality types. This fixes [#15214](https://github.com/ClickHouse/ClickHouse/issues/15214). [#15220](https://github.com/ClickHouse/ClickHouse/pull/15220) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#15487](https://github.com/ClickHouse/ClickHouse/issues/15487): Fix 'Unknown identifier' in GROUP BY when query has JOIN over Merge table. [#15242](https://github.com/ClickHouse/ClickHouse/pull/15242) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15282](https://github.com/ClickHouse/ClickHouse/issues/15282): Fix MSan report in QueryLog. Uninitialized memory can be used for the field `memory_usage`. [#15258](https://github.com/ClickHouse/ClickHouse/pull/15258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15401](https://github.com/ClickHouse/ClickHouse/issues/15401): Fix hang of queries with a lot of subqueries to same table of `MySQL` engine. Previously, if there were more than 16 subqueries to same `MySQL` table in query, it hang forever. [#15299](https://github.com/ClickHouse/ClickHouse/pull/15299) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#15340](https://github.com/ClickHouse/ClickHouse/issues/15340): Fix rare race condition on server startup when system.logs are enabled. [#15300](https://github.com/ClickHouse/ClickHouse/pull/15300) ([alesapin](https://github.com/alesapin)). +* Backported in [#15336](https://github.com/ClickHouse/ClickHouse/issues/15336): Fix race condition during MergeTree table rename and background cleanup. [#15304](https://github.com/ClickHouse/ClickHouse/pull/15304) ([alesapin](https://github.com/alesapin)). +* Backported in [#15588](https://github.com/ClickHouse/ClickHouse/issues/15588): Fix bug where queries like SELECT toStartOfDay(today()) fail complaining about empty time_zone argument. [#15319](https://github.com/ClickHouse/ClickHouse/pull/15319) ([Bharat Nallan](https://github.com/bharatnc)). +* Backported in [#15446](https://github.com/ClickHouse/ClickHouse/issues/15446): Report proper error when the second argument of `boundingRatio` aggregate function has a wrong type. [#15407](https://github.com/ClickHouse/ClickHouse/pull/15407) ([detailyang](https://github.com/detailyang)). +* Backported in [#15507](https://github.com/ClickHouse/ClickHouse/issues/15507): Fix bug with event subscription in DDLWorker which rarely may lead to query hangs in `ON CLUSTER`. Introduced in [#13450](https://github.com/ClickHouse/ClickHouse/issues/13450). [#15477](https://github.com/ClickHouse/ClickHouse/pull/15477) ([alesapin](https://github.com/alesapin)). +* Backported in [#15549](https://github.com/ClickHouse/ClickHouse/issues/15549): Fix `Missing columns` errors when selecting columns which absent in data, but depend on other columns which also absent in data. Fixes [#15530](https://github.com/ClickHouse/ClickHouse/issues/15530). [#15532](https://github.com/ClickHouse/ClickHouse/pull/15532) ([alesapin](https://github.com/alesapin)). +* Backported in [#15560](https://github.com/ClickHouse/ClickHouse/issues/15560): Fix bug when `ILIKE` operator stops being case insensitive if `LIKE` with the same pattern was executed. [#15536](https://github.com/ClickHouse/ClickHouse/pull/15536) ([alesapin](https://github.com/alesapin)). +* Backported in [#15725](https://github.com/ClickHouse/ClickHouse/issues/15725): Mutation might hang waiting for some non-existent part after `MOVE` or `REPLACE PARTITION` or, in rare cases, after `DETACH` or `DROP PARTITION`. It's fixed. [#15537](https://github.com/ClickHouse/ClickHouse/pull/15537) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15655](https://github.com/ClickHouse/ClickHouse/issues/15655): Fix 'Database doesn't exist.' in queries with IN and Distributed table when there's no database on initiator. [#15538](https://github.com/ClickHouse/ClickHouse/pull/15538) ([Artem Zuikov](https://github.com/4ertus2)). +* Backported in [#15631](https://github.com/ClickHouse/ClickHouse/issues/15631): Significantly reduce memory usage in AggregatingInOrderTransform/optimize_aggregation_in_order. [#15543](https://github.com/ClickHouse/ClickHouse/pull/15543) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#15583](https://github.com/ClickHouse/ClickHouse/issues/15583): Prevent the possibility of error message `Could not calculate available disk space (statvfs), errno: 4, strerror: Interrupted system call`. This fixes [#15541](https://github.com/ClickHouse/ClickHouse/issues/15541). [#15557](https://github.com/ClickHouse/ClickHouse/pull/15557) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#15665](https://github.com/ClickHouse/ClickHouse/issues/15665): Fixed `Element ... is not a constant expression` error when using `JSON*` function result in `VALUES`, `LIMIT` or right side of `IN` operator. [#15589](https://github.com/ClickHouse/ClickHouse/pull/15589) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15712](https://github.com/ClickHouse/ClickHouse/issues/15712): Fix the order of destruction for resources in `ReadFromStorage` step of query plan. It might cause crashes in rare cases. Possibly connected with [#15610](https://github.com/ClickHouse/ClickHouse/issues/15610). [#15645](https://github.com/ClickHouse/ClickHouse/pull/15645) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#15696](https://github.com/ClickHouse/ClickHouse/issues/15696): Fix race condition in AMQP-CPP. [#15667](https://github.com/ClickHouse/ClickHouse/pull/15667) ([alesapin](https://github.com/alesapin)). +* Backported in [#15739](https://github.com/ClickHouse/ClickHouse/issues/15739): Fix error `Cannot find column` which may happen at insertion into `MATERIALIZED VIEW` in case if query for `MV` containes `ARRAY JOIN`. [#15717](https://github.com/ClickHouse/ClickHouse/pull/15717) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v20.9.4.76-stable.md b/docs/changelogs/v20.9.4.76-stable.md new file mode 100644 index 00000000000..f1ec2252bc2 --- /dev/null +++ b/docs/changelogs/v20.9.4.76-stable.md @@ -0,0 +1,37 @@ +### ClickHouse release v20.9.4.76-stable FIXME as compared to v20.9.3.45-stable + +#### Improvement +* Backported in [#16145](https://github.com/ClickHouse/ClickHouse/issues/16145): Now it's allowed to execute `ALTER ... ON CLUSTER` queries regardless of the `` setting in cluster config. [#16075](https://github.com/ClickHouse/ClickHouse/pull/16075) ([alesapin](https://github.com/alesapin)). +* Backported in [#16312](https://github.com/ClickHouse/ClickHouse/issues/16312): Add allow_nondeterministic_optimize_skip_unused_shards (to allow non deterministic like rand() or dictGet() in sharding key). [#16105](https://github.com/ClickHouse/ClickHouse/pull/16105) ([Azat Khuzhin](https://github.com/azat)). + +#### Bug Fix +* Backported in [#15618](https://github.com/ClickHouse/ClickHouse/issues/15618): Throw an error when a single parameter is passed to ReplicatedMergeTree instead of ignoring it. [#15516](https://github.com/ClickHouse/ClickHouse/pull/15516) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#16201](https://github.com/ClickHouse/ClickHouse/issues/16201): Decrement the `ReadonlyReplica` metric when detaching read-only tables. This fixes [#15598](https://github.com/ClickHouse/ClickHouse/issues/15598). [#15592](https://github.com/ClickHouse/ClickHouse/pull/15592) ([sundyli](https://github.com/sundy-li)). +* Backported in [#16229](https://github.com/ClickHouse/ClickHouse/issues/16229): Fixed bug with globs in S3 table function, region from URL was not applied to S3 client configuration. [#15646](https://github.com/ClickHouse/ClickHouse/pull/15646) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#15870](https://github.com/ClickHouse/ClickHouse/issues/15870): Fix error `Cannot add simple transform to empty Pipe` which happened while reading from `Buffer` table which has different structure than destination table. It was possible if destination table returned empty result for query. Fixes [#15529](https://github.com/ClickHouse/ClickHouse/issues/15529). [#15662](https://github.com/ClickHouse/ClickHouse/pull/15662) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#15774](https://github.com/ClickHouse/ClickHouse/issues/15774): Fixed too low default value of `max_replicated_logs_to_keep` setting, which might cause replicas to become lost too often. Improve lost replica recovery process by choosing the most up-to-date replica to clone. Also do not remove old parts from lost replica, detach them instead. [#15701](https://github.com/ClickHouse/ClickHouse/pull/15701) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15798](https://github.com/ClickHouse/ClickHouse/issues/15798): Fix some cases of queries, in which only virtual columns are selected. Previously `Not found column _nothing in block` exception may be thrown. Fixes [#12298](https://github.com/ClickHouse/ClickHouse/issues/12298). [#15756](https://github.com/ClickHouse/ClickHouse/pull/15756) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#15868](https://github.com/ClickHouse/ClickHouse/issues/15868): Fix `select count()` inaccuracy for MaterializeMySQL. [#15767](https://github.com/ClickHouse/ClickHouse/pull/15767) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15901](https://github.com/ClickHouse/ClickHouse/issues/15901): Fix exception `Block structure mismatch` in `SELECT ... ORDER BY DESC` queries which were executed after `ALTER MODIFY COLUMN` query. Fixes [#15800](https://github.com/ClickHouse/ClickHouse/issues/15800). [#15852](https://github.com/ClickHouse/ClickHouse/pull/15852) ([alesapin](https://github.com/alesapin)). +* Backported in [#15924](https://github.com/ClickHouse/ClickHouse/issues/15924): Now exception will be thrown when `ALTER MODIFY COLUMN ... DEFAULT ...` has incompatible default with column type. Fixes [#15854](https://github.com/ClickHouse/ClickHouse/issues/15854). [#15858](https://github.com/ClickHouse/ClickHouse/pull/15858) ([alesapin](https://github.com/alesapin)). +* Backported in [#15919](https://github.com/ClickHouse/ClickHouse/issues/15919): Fix possible deadlocks in RBAC. [#15875](https://github.com/ClickHouse/ClickHouse/pull/15875) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#16167](https://github.com/ClickHouse/ClickHouse/issues/16167): Fix incorrect empty result for query from `Distributed` table if query has `WHERE`, `PREWHERE` and `GLOBAL IN`. Fixes [#15792](https://github.com/ClickHouse/ClickHouse/issues/15792). [#15933](https://github.com/ClickHouse/ClickHouse/pull/15933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#16356](https://github.com/ClickHouse/ClickHouse/issues/16356): Fixed `DROP TABLE IF EXISTS` failure with `Table ... doesn't exist` error when table is concurrently renamed (for Atomic database engine). Fixed rare deadlock when concurrently executing some DDL queries with multiple tables (like `DROP DATABASE` and `RENAME TABLE`) Fixed `DROP/DETACH DATABASE` failure with `Table ... doesn't exist` when concurrently executing `DROP/DETACH TABLE`. [#15934](https://github.com/ClickHouse/ClickHouse/pull/15934) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#15971](https://github.com/ClickHouse/ClickHouse/issues/15971): Fix a crash when database creation fails. [#15954](https://github.com/ClickHouse/ClickHouse/pull/15954) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#16022](https://github.com/ClickHouse/ClickHouse/issues/16022): Fix ambiguity in parsing of settings profiles: `CREATE USER ... SETTINGS profile readonly` is now considered as using a profile named `readonly`, not a setting named `profile` with the readonly constraint. This fixes [#15628](https://github.com/ClickHouse/ClickHouse/issues/15628). [#15982](https://github.com/ClickHouse/ClickHouse/pull/15982) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#16218](https://github.com/ClickHouse/ClickHouse/issues/16218): Fix rare segfaults when inserting into or selecting from MaterializedView and concurrently dropping target table (for Atomic database engine). [#15984](https://github.com/ClickHouse/ClickHouse/pull/15984) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#16026](https://github.com/ClickHouse/ClickHouse/issues/16026): Prevent replica hang for 5-10 mins when replication error happens after a period of inactivity. [#15987](https://github.com/ClickHouse/ClickHouse/pull/15987) ([filimonov](https://github.com/filimonov)). +* Backported in [#16091](https://github.com/ClickHouse/ClickHouse/issues/16091): Allow to use direct layout for dictionaries with complex keys. [#16007](https://github.com/ClickHouse/ClickHouse/pull/16007) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#16361](https://github.com/ClickHouse/ClickHouse/issues/16361): Fix collate name & charset name parser and support `length = 0` for string type. [#16008](https://github.com/ClickHouse/ClickHouse/pull/16008) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#16140](https://github.com/ClickHouse/ClickHouse/issues/16140): Fix `ALTER MODIFY ... ORDER BY` query hang for `ReplicatedVersionedCollapsingMergeTree`. This fixes [#15980](https://github.com/ClickHouse/ClickHouse/issues/15980). [#16011](https://github.com/ClickHouse/ClickHouse/pull/16011) ([alesapin](https://github.com/alesapin)). +* Backported in [#16079](https://github.com/ClickHouse/ClickHouse/issues/16079): Fixes [#15780](https://github.com/ClickHouse/ClickHouse/issues/15780) regression, e.g. indexOf([1, 2, 3], toLowCardinality(1)) now is prohibited but it should not be. [#16038](https://github.com/ClickHouse/ClickHouse/pull/16038) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#16298](https://github.com/ClickHouse/ClickHouse/issues/16298): Fix dictGet in sharding_key (and similar places, i.e. when the function context is stored permanently). [#16205](https://github.com/ClickHouse/ClickHouse/pull/16205) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16232](https://github.com/ClickHouse/ClickHouse/issues/16232): Fix the case when memory can be overallocated regardless to the limit. This closes [#14560](https://github.com/ClickHouse/ClickHouse/issues/14560). [#16206](https://github.com/ClickHouse/ClickHouse/pull/16206) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#16326](https://github.com/ClickHouse/ClickHouse/issues/16326): Fix a possible memory leak during `GROUP BY` with string keys, caused by an error in `TwoLevelStringHashTable` implementation. [#16264](https://github.com/ClickHouse/ClickHouse/pull/16264) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#16377](https://github.com/ClickHouse/ClickHouse/issues/16377): Fix async Distributed INSERT w/ prefer_localhost_replica=0 and internal_replication. [#16358](https://github.com/ClickHouse/ClickHouse/pull/16358) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16432](https://github.com/ClickHouse/ClickHouse/issues/16432): Fix group by with totals/rollup/cube modifers and min/max functions over group by keys. Fixes [#16393](https://github.com/ClickHouse/ClickHouse/issues/16393). [#16397](https://github.com/ClickHouse/ClickHouse/pull/16397) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#16449](https://github.com/ClickHouse/ClickHouse/issues/16449): Fix double free in case of exception in function `dictGet`. It could have happened if dictionary was loaded with error. [#16429](https://github.com/ClickHouse/ClickHouse/pull/16429) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Other +* Unfold `{database}`, `{table}` and `{uuid}` macros in `ReplicatedMergeTree` arguments on table creation. [#16160](https://github.com/ClickHouse/ClickHouse/pull/16160) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v20.9.5.5-stable.md b/docs/changelogs/v20.9.5.5-stable.md new file mode 100644 index 00000000000..d200972e071 --- /dev/null +++ b/docs/changelogs/v20.9.5.5-stable.md @@ -0,0 +1,12 @@ +### ClickHouse release v20.9.5.5-stable FIXME as compared to v20.9.4.76-stable + +#### Bug Fix +* Backported in [#16493](https://github.com/ClickHouse/ClickHouse/issues/16493): Fix bug with MySQL database. When MySQL server used as database engine is down some queries raise Exception, because they try to get tables from disabled server, while it's unnecessary. For example, query `SELECT ... FROM system.parts` should work only with MergeTree tables and don't touch MySQL database at all. [#16032](https://github.com/ClickHouse/ClickHouse/pull/16032) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#16817](https://github.com/ClickHouse/ClickHouse/issues/16817): Fixed the inconsistent behaviour when a part of return data could be dropped because the set for its filtration wasn't created. [#16308](https://github.com/ClickHouse/ClickHouse/pull/16308) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#16508](https://github.com/ClickHouse/ClickHouse/issues/16508): Fix processing of very large entries in replication queue. Very large entries may appear in ALTER queries if table structure is extremely large (near 1 MB). This fixes [#16307](https://github.com/ClickHouse/ClickHouse/issues/16307). [#16332](https://github.com/ClickHouse/ClickHouse/pull/16332) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#16474](https://github.com/ClickHouse/ClickHouse/issues/16474): Fix DROP TABLE for Distributed (racy with INSERT). [#16409](https://github.com/ClickHouse/ClickHouse/pull/16409) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16571](https://github.com/ClickHouse/ClickHouse/issues/16571): Fix rapid growth of metadata when using MySQL Master -> MySQL Slave -> ClickHouse MaterializeMySQL Engine, and `slave_parallel_worker` enabled on MySQL Slave, by properly shrinking GTID sets. This fixes [#15951](https://github.com/ClickHouse/ClickHouse/issues/15951). [#16504](https://github.com/ClickHouse/ClickHouse/pull/16504) ([TCeason](https://github.com/TCeason)). +* Backported in [#16554](https://github.com/ClickHouse/ClickHouse/issues/16554): Now when parsing AVRO from input the LowCardinality is removed from type. Fixes [#16188](https://github.com/ClickHouse/ClickHouse/issues/16188). [#16521](https://github.com/ClickHouse/ClickHouse/pull/16521) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#16748](https://github.com/ClickHouse/ClickHouse/issues/16748): Fixed [#16081](https://github.com/ClickHouse/ClickHouse/issues/16081). [#16613](https://github.com/ClickHouse/ClickHouse/pull/16613) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#16894](https://github.com/ClickHouse/ClickHouse/issues/16894): Fix rare silent crashes when query profiler is on and ClickHouse is installed on OS with glibc version that has (supposedly) broken asynchronous unwind tables for some functions. This fixes [#15301](https://github.com/ClickHouse/ClickHouse/issues/15301). This fixes [#13098](https://github.com/ClickHouse/ClickHouse/issues/13098). [#16846](https://github.com/ClickHouse/ClickHouse/pull/16846) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v20.9.6.14-stable.md b/docs/changelogs/v20.9.6.14-stable.md new file mode 100644 index 00000000000..05fdfcc30aa --- /dev/null +++ b/docs/changelogs/v20.9.6.14-stable.md @@ -0,0 +1,19 @@ +### ClickHouse release v20.9.6.14-stable FIXME as compared to v20.9.5.5-stable + +#### Improvement +* Backported in [#17030](https://github.com/ClickHouse/ClickHouse/issues/17030): Make it possible to connect to `clickhouse-server` secure endpoint which requires SNI. This is possible when `clickhouse-server` is hosted behind TLS proxy. [#16938](https://github.com/ClickHouse/ClickHouse/pull/16938) ([filimonov](https://github.com/filimonov)). + +#### Bug Fix +* Backported in [#15678](https://github.com/ClickHouse/ClickHouse/issues/15678): Query is finished faster in case of exception. Cancel execution on remote replicas if exception happens. [#15578](https://github.com/ClickHouse/ClickHouse/pull/15578) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#17105](https://github.com/ClickHouse/ClickHouse/issues/17105): fixes [#16574](https://github.com/ClickHouse/ClickHouse/issues/16574) fixes [#16231](https://github.com/ClickHouse/ClickHouse/issues/16231) fix remote query failure when using 'if' suffix aggregate function. [#16610](https://github.com/ClickHouse/ClickHouse/pull/16610) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#16759](https://github.com/ClickHouse/ClickHouse/issues/16759): This will fix optimize_read_in_order/optimize_aggregation_in_order with max_threads>0 and expression in ORDER BY. [#16637](https://github.com/ClickHouse/ClickHouse/pull/16637) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#16740](https://github.com/ClickHouse/ClickHouse/issues/16740): Fix `IN` operator over several columns and tuples with enabled `transform_null_in` setting. Fixes [#15310](https://github.com/ClickHouse/ClickHouse/issues/15310). [#16722](https://github.com/ClickHouse/ClickHouse/pull/16722) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17023](https://github.com/ClickHouse/ClickHouse/issues/17023): Fix crash when using `any` without any arguments. This is for [#16803](https://github.com/ClickHouse/ClickHouse/issues/16803) . cc @azat. [#16826](https://github.com/ClickHouse/ClickHouse/pull/16826) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#16879](https://github.com/ClickHouse/ClickHouse/issues/16879): Abort multipart upload if no data was written to WriteBufferFromS3. [#16840](https://github.com/ClickHouse/ClickHouse/pull/16840) ([Pavel Kovalenko](https://github.com/Jokser)). +* Backported in [#16945](https://github.com/ClickHouse/ClickHouse/issues/16945): Prevent clickhouse server crashes when using TimeSeriesGroupSum. [#16865](https://github.com/ClickHouse/ClickHouse/pull/16865) ([filimonov](https://github.com/filimonov)). +* Backported in [#17078](https://github.com/ClickHouse/ClickHouse/issues/17078): Fix possible error `Illegal type of argument` for queries with `ORDER BY`. Fixes [#16580](https://github.com/ClickHouse/ClickHouse/issues/16580). [#16928](https://github.com/ClickHouse/ClickHouse/pull/16928) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17009](https://github.com/ClickHouse/ClickHouse/issues/17009): Install script should always create subdirs in config folders. This is only relevant for Docker build with custom config. [#16936](https://github.com/ClickHouse/ClickHouse/pull/16936) ([filimonov](https://github.com/filimonov)). +* Backported in [#17013](https://github.com/ClickHouse/ClickHouse/issues/17013): Fix possible server crash after `ALTER TABLE ... MODIFY COLUMN ... NewType` when `SELECT` have `WHERE` expression on altering column and alter doesn't finished yet. [#16968](https://github.com/ClickHouse/ClickHouse/pull/16968) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17037](https://github.com/ClickHouse/ClickHouse/issues/17037): Reresolve the IP of the `format_avro_schema_registry_url` in case of errors. [#16985](https://github.com/ClickHouse/ClickHouse/pull/16985) ([filimonov](https://github.com/filimonov)). +* Backported in [#17172](https://github.com/ClickHouse/ClickHouse/issues/17172): Fix bug when `ON CLUSTER` queries may hang forever for non-leader ReplicatedMergeTreeTables. [#17089](https://github.com/ClickHouse/ClickHouse/pull/17089) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v20.9.7.11-stable.md b/docs/changelogs/v20.9.7.11-stable.md new file mode 100644 index 00000000000..e5b35a3eb32 --- /dev/null +++ b/docs/changelogs/v20.9.7.11-stable.md @@ -0,0 +1,27 @@ +### ClickHouse release v20.9.7.11-stable FIXME as compared to v20.9.6.14-stable + +#### Performance Improvement +* Backported in [#17590](https://github.com/ClickHouse/ClickHouse/issues/17590): Fix performance of reading from `Merge` tables over huge number of `MergeTree` tables. Fixes [#7748](https://github.com/ClickHouse/ClickHouse/issues/7748). [#16988](https://github.com/ClickHouse/ClickHouse/pull/16988) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix +* Backported in [#17316](https://github.com/ClickHouse/ClickHouse/issues/17316): Return number of affected rows for INSERT queries via MySQL protocol. Previously ClickHouse used to always return 0, it's fixed. Fixes [#16605](https://github.com/ClickHouse/ClickHouse/issues/16605). [#16715](https://github.com/ClickHouse/ClickHouse/pull/16715) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#17343](https://github.com/ClickHouse/ClickHouse/issues/17343): TODO. [#16866](https://github.com/ClickHouse/ClickHouse/pull/16866) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17196](https://github.com/ClickHouse/ClickHouse/issues/17196): Avoid unnecessary network errors for remote queries which may be cancelled while execution, like queries with `LIMIT`. [#17006](https://github.com/ClickHouse/ClickHouse/pull/17006) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#17432](https://github.com/ClickHouse/ClickHouse/issues/17432): Bug fix for funciton fuzzBits, related issue: [#16980](https://github.com/ClickHouse/ClickHouse/issues/16980). [#17051](https://github.com/ClickHouse/ClickHouse/pull/17051) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#17129](https://github.com/ClickHouse/ClickHouse/issues/17129): Fixed crash on `CREATE TABLE ... AS some_table` query when `some_table` was created `AS table_function()` Fixes [#16944](https://github.com/ClickHouse/ClickHouse/issues/16944). [#17072](https://github.com/ClickHouse/ClickHouse/pull/17072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17195](https://github.com/ClickHouse/ClickHouse/issues/17195): Fix ColumnConst comparison which leads to crash. This fixed [#17088](https://github.com/ClickHouse/ClickHouse/issues/17088) . [#17135](https://github.com/ClickHouse/ClickHouse/pull/17135) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17402](https://github.com/ClickHouse/ClickHouse/issues/17402): Fix [#15235](https://github.com/ClickHouse/ClickHouse/issues/15235). When clickhouse-copier handle non-partitioned table, throws segfault error. [#17248](https://github.com/ClickHouse/ClickHouse/pull/17248) ([Qi Chen](https://github.com/kaka11chen)). +* Backported in [#17409](https://github.com/ClickHouse/ClickHouse/issues/17409): Fix set index invalidation when there are const columns in the subquery. This fixes [#17246](https://github.com/ClickHouse/ClickHouse/issues/17246) . [#17249](https://github.com/ClickHouse/ClickHouse/pull/17249) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#17488](https://github.com/ClickHouse/ClickHouse/issues/17488): Fix crash while reading from `JOIN` table with `LowCardinality` types. Fixes [#17228](https://github.com/ClickHouse/ClickHouse/issues/17228). [#17397](https://github.com/ClickHouse/ClickHouse/pull/17397) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17492](https://github.com/ClickHouse/ClickHouse/issues/17492): Fix duplicates after `DISTINCT` which were possible because of incorrect optimization. Fixes [#17294](https://github.com/ClickHouse/ClickHouse/issues/17294). [#17296](https://github.com/ClickHouse/ClickHouse/pull/17296) ([li chengxiang](https://github.com/chengxianglibra)). [#17439](https://github.com/ClickHouse/ClickHouse/pull/17439) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#17522](https://github.com/ClickHouse/ClickHouse/issues/17522): Fix `ORDER BY` with enabled setting `optimize_redundant_functions_in_order_by`. [#17471](https://github.com/ClickHouse/ClickHouse/pull/17471) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#17535](https://github.com/ClickHouse/ClickHouse/issues/17535): Fix bug when mark cache size was underestimated by clickhouse. It may happen when there are a lot of tiny files with marks. [#17496](https://github.com/ClickHouse/ClickHouse/pull/17496) ([alesapin](https://github.com/alesapin)). +* Backported in [#17627](https://github.com/ClickHouse/ClickHouse/issues/17627): Fix alter query hang when the corresponding mutation was killed on the different replica. Fixes [#16953](https://github.com/ClickHouse/ClickHouse/issues/16953). [#17499](https://github.com/ClickHouse/ClickHouse/pull/17499) ([alesapin](https://github.com/alesapin)). +* Backported in [#17608](https://github.com/ClickHouse/ClickHouse/issues/17608): When clickhouse-client is used in interactive mode with multiline queries, single line comment was erronously extended till the end of query. This fixes [#13654](https://github.com/ClickHouse/ClickHouse/issues/13654). [#17565](https://github.com/ClickHouse/ClickHouse/pull/17565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#17729](https://github.com/ClickHouse/ClickHouse/issues/17729): Fixed `Function not implemented` error when executing `RENAME` query in `Atomic` database with ClickHouse running on Windows Subsystem for Linux. Fixes [#17661](https://github.com/ClickHouse/ClickHouse/issues/17661). [#17664](https://github.com/ClickHouse/ClickHouse/pull/17664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#17782](https://github.com/ClickHouse/ClickHouse/issues/17782): Fixed problem when ClickHouse fails to resume connection to MySQL servers. [#17681](https://github.com/ClickHouse/ClickHouse/pull/17681) ([Alexander Kazakov](https://github.com/Akazz)). +* Backported in [#17814](https://github.com/ClickHouse/ClickHouse/issues/17814): Do not restore parts from WAL if `in_memory_parts_enable_wal` is disabled. [#17802](https://github.com/ClickHouse/ClickHouse/pull/17802) ([detailyang](https://github.com/detailyang)). + +#### Build/Testing/Packaging Improvement +* Backported in [#17288](https://github.com/ClickHouse/ClickHouse/issues/17288): Update embedded timezone data to version 2020d (also update cctz to the latest master). [#17204](https://github.com/ClickHouse/ClickHouse/pull/17204) ([filimonov](https://github.com/filimonov)). + diff --git a/docs/changelogs/v21.1.1.5646-prestable.md b/docs/changelogs/v21.1.1.5646-prestable.md new file mode 100644 index 00000000000..ec8abc8a05b --- /dev/null +++ b/docs/changelogs/v21.1.1.5646-prestable.md @@ -0,0 +1,259 @@ +### ClickHouse release v21.1.1.5646-prestable FIXME as compared to v20.12.1.5236-prestable + +#### Backward Incompatible Change +* Prohibit toUnixTimestamp(Date()) (before it just returns UInt16 representation of Date). [#17376](https://github.com/ClickHouse/ClickHouse/pull/17376) ([Azat Khuzhin](https://github.com/azat)). +* Removed aggregate functions `timeSeriesGroupSum`, `timeSeriesGroupRateSum` because a friend of mine said they never worked. This fixes [#16869](https://github.com/ClickHouse/ClickHouse/issues/16869). If you have luck using these functions, write a email to clickhouse-feedback@yandex-team.com. [#17423](https://github.com/ClickHouse/ClickHouse/pull/17423) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The setting `input_format_null_as_default` is enabled by default. [#17525](https://github.com/ClickHouse/ClickHouse/pull/17525) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Set `insert_quorum_parallel` to 1 by default. It is significantly more convenient to use than "sequential" quorum inserts. But if you rely to sequential consistency, you should set the setting back to zero. [#17567](https://github.com/ClickHouse/ClickHouse/pull/17567) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Restrict `ALTER MODIFY SETTING` from changing storage settings that affects data parts (`write_final_mark` and `enable_mixed_granularity_parts`). [#18306](https://github.com/ClickHouse/ClickHouse/pull/18306) ([Amos Bird](https://github.com/amosbird)). +* Check settings constraints for profile settings from config. Server will fail to start if users.xml contain settings that do not meet corresponding constraints. [#18486](https://github.com/ClickHouse/ClickHouse/pull/18486) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove `sumburConsistentHash` function. This closes [#18120](https://github.com/ClickHouse/ClickHouse/issues/18120). [#18656](https://github.com/ClickHouse/ClickHouse/pull/18656) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `DIV` and `MOD` operators. `DIV` implements integer division. This is needed for MySQL compatibility and for `sqllogictest`. This closes [#18707](https://github.com/ClickHouse/ClickHouse/issues/18707). It may introduce incompatibilities if you are using DIV or MOD as column names or aliases. In case of incompatibility, write aliases after AS keyword or wrap identifiers in quotes (either double quotes or backquotes). [#18760](https://github.com/ClickHouse/ClickHouse/pull/18760) ([Du Chuan](https://github.com/spongedu)). + +#### New Feature +* Use https://github.com/lemire/fast_float to parse floating point numbers. [#16787](https://github.com/ClickHouse/ClickHouse/pull/16787) ([Maksim Kita](https://github.com/kitaisreal)). +* ... [#16819](https://github.com/ClickHouse/ClickHouse/pull/16819) ([pronvis](https://github.com/pronvis)). +* Provide a new aggregator combinator : `-SimpleState` to build SimpleAggregateFunction types via query. It's useful for defining MaterializedView of AggregatingMergeTree engine, and will benefit projections too. [#16853](https://github.com/ClickHouse/ClickHouse/pull/16853) ([Amos Bird](https://github.com/amosbird)). +* Added `mannWitneyUTest`, `studentTTest` and `welchTTest` aggregate functions. Refactored RankCorr a bit. [#16883](https://github.com/ClickHouse/ClickHouse/pull/16883) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add a setting optimize_on_insert. When enabled, do the same transformation for INSERTed block of data as if merge was done on this block (e.g. Replacing, Collapsing, Aggregating...). This setting will be enabled as default. This can influence Materialized View and MaterializeMySQL behaviour (see detailed description). This closes [#10683](https://github.com/ClickHouse/ClickHouse/issues/10683). [#16954](https://github.com/ClickHouse/ClickHouse/pull/16954) ([Kruglov Pavel](https://github.com/Avogar)). +* add ```*.zst``` compression/decompression support.It enables using ```*.zst``` in ```file()``` function and ```Content-encoding: zstd``` in http client.This closes [#16791 ](https://github.com/ClickHouse/ClickHouse/issues/16791). [#17144](https://github.com/ClickHouse/ClickHouse/pull/17144) ([Abi Palagashvili](https://github.com/fibersel)). +* * IP Dictionary supports `IPv4` / `IPv6` types directly. [#17571](https://github.com/ClickHouse/ClickHouse/pull/17571) ([Vladimir C](https://github.com/vdimir)). +* related: [#16176](https://github.com/ClickHouse/ClickHouse/issues/16176) Usage: ``` set limit = 10; set offset = 20; ``` this two settings will affect SELECT query as if it is added like ``` select * from ($your_original_select_query) tmp limit xxx offset xxx; ```. [#17633](https://github.com/ClickHouse/ClickHouse/pull/17633) ([hexiaoting](https://github.com/hexiaoting)). +* Add asynchronous metrics on total amount of rows, bytes and parts in MergeTree tables. This fix [#11714](https://github.com/ClickHouse/ClickHouse/issues/11714). [#17639](https://github.com/ClickHouse/ClickHouse/pull/17639) ([flynn](https://github.com/ucasfl)). +* Introduce `DETACH TABLE/VIEW ... PERMANENTLY` syntax, so that after restarting the table does not reappear back automatically (only by explicit request). The table can still be attached back using the short syntax ATTACH TABLE. Implements [#5555](https://github.com/ClickHouse/ClickHouse/issues/5555). Fixes [#13850](https://github.com/ClickHouse/ClickHouse/issues/13850). [#17642](https://github.com/ClickHouse/ClickHouse/pull/17642) ([filimonov](https://github.com/filimonov)). +* Adds a new table called `system.distributed_ddl_queue` that displays the queries in the DDL worker queue. [#17656](https://github.com/ClickHouse/ClickHouse/pull/17656) ([Bharat Nallan](https://github.com/bharatnc)). +* Add function `encodeXMLComponent` to escape characters to place string into XML text node or attribute. [#17659](https://github.com/ClickHouse/ClickHouse/pull/17659) ([nauta](https://github.com/nautaa)). +* Now clickhouse-client supports opening EDITOR to edit commands. `Alt-Shift-E`. [#17665](https://github.com/ClickHouse/ClickHouse/pull/17665) ([Amos Bird](https://github.com/amosbird)). +* Add support for PROXYv1 protocol to wrap native TCP interface. Allow quotas to be keyed by proxy-forwarded IP address (applied for PROXYv1 address and for X-Forwarded-For from HTTP interface). This is useful when you provide access to ClickHouse only via trusted proxy (e.g. CloudFlare) but want to account user resources by their original IP addresses. This fixes [#17268](https://github.com/ClickHouse/ClickHouse/issues/17268). [#17707](https://github.com/ClickHouse/ClickHouse/pull/17707) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add ability to use custom TLD list in functions `firstSignificantSubdomainCustom`, `cutToFirstSignificantSubdomainCustom`. [#17748](https://github.com/ClickHouse/ClickHouse/pull/17748) ([Azat Khuzhin](https://github.com/azat)). +* Clickhouse-benchmark added query parameter. [#17832](https://github.com/ClickHouse/ClickHouse/pull/17832) ([Maksim Kita](https://github.com/kitaisreal)). +* Extended `OPTIMIZE ... DEDUPLICATE` syntax to allow explicit (or implicit with asterisk/column transformers) list of columns to check for duplicates on. ... [#17846](https://github.com/ClickHouse/ClickHouse/pull/17846) ([Vasily Nemkov](https://github.com/Enmk)). +* Add settings `min_compress_block_size` and `max_compress_block_size` to MergeTreeSettings, which have higher priority than the global settings and take effect when they are set. close [13890](https://github.com/ClickHouse/ClickHouse/issues/13890). [#17867](https://github.com/ClickHouse/ClickHouse/pull/17867) ([flynn](https://github.com/ucasfl)). +* Implemented `ATTACH TABLE name FROM 'path/to/data/' (col1 Type1, ...` query. It creates new table with provided structure and attaches table data from provided directory in `user_files`. [#17903](https://github.com/ClickHouse/ClickHouse/pull/17903) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Added `arrayMin`, `arrayMax`, `arrayAvg` aggregation functions. [#18032](https://github.com/ClickHouse/ClickHouse/pull/18032) ([Maksim Kita](https://github.com/kitaisreal)). +* Support `SHOW SETTINGS` statement to show parameters in system.settings. `SHOW CHANGED SETTINGS` and `LIKE/ILIKE` clause are also supported. [#18056](https://github.com/ClickHouse/ClickHouse/pull/18056) ([Jianmei Zhang](https://github.com/zhangjmruc)). +* Allow create table as select with columns specification. Example `CREATE TABLE t1 (x String) ENGINE = Memory AS SELECT 1;`. [#18060](https://github.com/ClickHouse/ClickHouse/pull/18060) ([Maksim Kita](https://github.com/kitaisreal)). +* - IP Dictionary supports key fetching. Resolves [#18241](https://github.com/ClickHouse/ClickHouse/issues/18241). [#18480](https://github.com/ClickHouse/ClickHouse/pull/18480) ([Vladimir C](https://github.com/vdimir)). +* Implemented `REPLACE TABLE` and `CREATE OR REPLACE TABLE` queries. [#18521](https://github.com/ClickHouse/ClickHouse/pull/18521) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Added function `byteSize` to estimate of uncompressed byte size of its arguments in memory. E.g. for UInt32 argument it will return constant 4, for String argument - the string length + 9. The function can take multiple arguments. The typical application is byteSize(*). [#18579](https://github.com/ClickHouse/ClickHouse/pull/18579) ([Ping Yu](https://github.com/pingyu)). +* Add `query_id` column to `system.part_log` for inserted parts. closes [#10097](https://github.com/ClickHouse/ClickHouse/issues/10097). [#18644](https://github.com/ClickHouse/ClickHouse/pull/18644) ([flynn](https://github.com/ucasfl)). +* Now we have a new storage setting `max_partitions_to_read` for tables in the MergeTree family. It limits the max number of partitions that can be accessed in one query. A user setting `force_max_partition_limit` is also added to enforce this constraint. [#18712](https://github.com/ClickHouse/ClickHouse/pull/18712) ([Amos Bird](https://github.com/amosbird)). +* Function `position` now supports `position(needle in haystack)` synax for SQL compatibility. This closes [#18701](https://github.com/ClickHouse/ClickHouse/issues/18701). ... [#18779](https://github.com/ClickHouse/ClickHouse/pull/18779) ([Jianmei Zhang](https://github.com/zhangjmruc)). + +#### Performance Improvement +* Slightly improved performance of float parsing. [#16809](https://github.com/ClickHouse/ClickHouse/pull/16809) ([Maksim Kita](https://github.com/kitaisreal)). +* Improved performance of function `repeat`. [#16937](https://github.com/ClickHouse/ClickHouse/pull/16937) ([satanson](https://github.com/satanson)). +* Fix performance of reading from `Merge` tables over huge number of `MergeTree` tables. Fixes [#7748](https://github.com/ClickHouse/ClickHouse/issues/7748). [#16988](https://github.com/ClickHouse/ClickHouse/pull/16988) ([Anton Popov](https://github.com/CurtizJ)). +* Now the `-If` combinator is devirtualized, and `count` is properly vectorized. This is for https://github.com/ClickHouse/ClickHouse/pull/17041. [#17043](https://github.com/ClickHouse/ClickHouse/pull/17043) ([Amos Bird](https://github.com/amosbird)). +* Improve performance of AggregatingMergeTree w/ SimpleAggregateFunction(String) in PK. [#17109](https://github.com/ClickHouse/ClickHouse/pull/17109) ([Azat Khuzhin](https://github.com/azat)). +* Add `remerge_sort_lowered_memory_bytes_ratio` setting (If memory usage after remerge does not reduced by this ratio, remerge will be disabled). [#17539](https://github.com/ClickHouse/ClickHouse/pull/17539) ([Azat Khuzhin](https://github.com/azat)). +* Speedup `IPv6CIDRToRange` implementation. [#17569](https://github.com/ClickHouse/ClickHouse/pull/17569) ([Vladimir C](https://github.com/vdimir)). +* Using dragonbox algorithm for float to string conversion instead of ryu. [#17831](https://github.com/ClickHouse/ClickHouse/pull/17831) ([Maksim Kita](https://github.com/kitaisreal)). +* Optimized read for StorageMemory. [#18052](https://github.com/ClickHouse/ClickHouse/pull/18052) ([Maksim Kita](https://github.com/kitaisreal)). +* Don't send empty blocks to shards on synchronous INSERT into Distributed table. This closes [#14571](https://github.com/ClickHouse/ClickHouse/issues/14571). [#18775](https://github.com/ClickHouse/ClickHouse/pull/18775) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace `PODArray` with `PODArrayWithStackMemory` in `AggregateFunctionWindowFunnelData` to improvement `windowFunnel` function performance. [#18817](https://github.com/ClickHouse/ClickHouse/pull/18817) ([flynn](https://github.com/ucasfl)). +* Add `--no-system-table` option for `clickhouse-local` to run without system tables. This avoids initialization of `DateLUT` that may take noticeable amount of time (tens of milliseconds) at startup. [#18899](https://github.com/ClickHouse/ClickHouse/pull/18899) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* - New IP Dictionary implementation with lower memory consumption, improved performance for some cases, and fixed bugs. [#16804](https://github.com/ClickHouse/ClickHouse/pull/16804) ([Vladimir C](https://github.com/vdimir)). +* Added proper authentication using environment, `~/.aws` and `AssumeRole` for S3 client. [#16856](https://github.com/ClickHouse/ClickHouse/pull/16856) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Support HTTP proxy and HTTPS S3 endpoint configuration. [#16861](https://github.com/ClickHouse/ClickHouse/pull/16861) ([Pavel Kovalenko](https://github.com/Jokser)). +* When `-OrNull` combinator combined `-If`, `-Merge`, `-MergeState`, `-State` combinators, we should put `-OrNull` in front. [#16935](https://github.com/ClickHouse/ClickHouse/pull/16935) ([flynn](https://github.com/ucasfl)). +* - Add configuration for multi zookeeper clusters. [#17070](https://github.com/ClickHouse/ClickHouse/pull/17070) ([fastio](https://github.com/fastio)). +* - Add limit for http redirects in request to S3 storage ('s3_max_redirects'). [#17220](https://github.com/ClickHouse/ClickHouse/pull/17220) ([ianton-ru](https://github.com/ianton-ru)). +* Now set indices will work with `GLOBAL IN`. This fixes [#17232](https://github.com/ClickHouse/ClickHouse/issues/17232) , [#5576](https://github.com/ClickHouse/ClickHouse/issues/5576) . [#17253](https://github.com/ClickHouse/ClickHouse/pull/17253) ([Amos Bird](https://github.com/amosbird)). +* Avoid possible stack overflow in bigint conversion. Big integers are experimental. [#17269](https://github.com/ClickHouse/ClickHouse/pull/17269) ([flynn](https://github.com/ucasfl)). +* Improved minimal Web UI: add history; add sharing support; avoid race condition of different requests; add request in-flight and ready indicators; add favicon; detect Ctrl+Enter if textarea is not in focus. [#17293](https://github.com/ClickHouse/ClickHouse/pull/17293) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to use `optimize_move_to_prewhere` optimization with compact parts, when sizes of columns are unknown. [#17330](https://github.com/ClickHouse/ClickHouse/pull/17330) ([Anton Popov](https://github.com/CurtizJ)). +* Implement `countSubstrings()`/`countSubstringsCaseInsensitive()`/`countSubstringsCaseInsensitiveUTF8()` (Count the number of substring occurrences). [#17347](https://github.com/ClickHouse/ClickHouse/pull/17347) ([Azat Khuzhin](https://github.com/azat)). +* Add eof check in receiveHello to prevent getting `Attempt to read after eof` exception. [#17365](https://github.com/ClickHouse/ClickHouse/pull/17365) ([Kruglov Pavel](https://github.com/Avogar)). +* Replaced `malloc` with `new`, so that the `MemoryTracker` takes this memory into account. [#17412](https://github.com/ClickHouse/ClickHouse/pull/17412) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix parsing of SETTINGS clause of the INSERT ... SELECT ... SETTINGS query. [#17414](https://github.com/ClickHouse/ClickHouse/pull/17414) ([Azat Khuzhin](https://github.com/azat)). +* Multiple improvements in `./clickhouse install` script. [#17421](https://github.com/ClickHouse/ClickHouse/pull/17421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Simplify Sys/V init script. It was not working on Ubuntu 12.04. [#17428](https://github.com/ClickHouse/ClickHouse/pull/17428) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now queries coming to the server via MySQL and PostgreSQL protocols have distinctive interface types (which can be seen in the `interface` column of the table`system.query_log`): `4` for MySQL, and `5` for PostgreSQL, instead of formerly used `1` which is now used for the native protocol only. [#17437](https://github.com/ClickHouse/ClickHouse/pull/17437) ([Vitaly Baranov](https://github.com/vitlibar)). +* Allow specifying [TTL](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl) to remove old entries from [system log tables](https://clickhouse.tech/docs/en/operations/system-tables/), using the `` attribute in `config.xml`. [#17438](https://github.com/ClickHouse/ClickHouse/pull/17438) ([Du Chuan](https://github.com/spongedu)). +* Add functions countMatches/countMatchesCaseInsensitive. [#17459](https://github.com/ClickHouse/ClickHouse/pull/17459) ([Azat Khuzhin](https://github.com/azat)). +* Return dynamic columns like MATERIALIZED / ALIAS for wildcard query when switches `asterisk_include_materialized_columns` and `asterisk_include_alias_columns` are turned on. [#17462](https://github.com/ClickHouse/ClickHouse/pull/17462) ([Ken Chen](https://github.com/chenziliang)). +* Export asynchronous metrics of all servers current threads. It's useful to track down issues like https://github.com/ClickHouse-Extras/poco/pull/28. [#17463](https://github.com/ClickHouse/ClickHouse/pull/17463) ([Amos Bird](https://github.com/amosbird)). +* Export current max ddl entry executed by DDLWorker. It's useful to check if DDLWorker hangs somewhere. [#17464](https://github.com/ClickHouse/ClickHouse/pull/17464) ([Amos Bird](https://github.com/amosbird)). +* Query obfuscator: avoid usage of some SQL keywords for identifier names. [#17526](https://github.com/ClickHouse/ClickHouse/pull/17526) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow query parameters in UPDATE statement of ALTER query. Fixes [#10976](https://github.com/ClickHouse/ClickHouse/issues/10976). [#17563](https://github.com/ClickHouse/ClickHouse/pull/17563) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't throw "Too many parts" error in the middle of INSERT query. [#17566](https://github.com/ClickHouse/ClickHouse/pull/17566) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to reload symbols from debug file. This PR also fixes a build-id issue. [#17637](https://github.com/ClickHouse/ClickHouse/pull/17637) ([Amos Bird](https://github.com/amosbird)). +* This fixes [#17457](https://github.com/ClickHouse/ClickHouse/issues/17457). [#17641](https://github.com/ClickHouse/ClickHouse/pull/17641) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Switch to patched version of RocksDB (from ClickHouse-Extras). [#17643](https://github.com/ClickHouse/ClickHouse/pull/17643) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Do not ignore server memory limits during Buffer flush. [#17646](https://github.com/ClickHouse/ClickHouse/pull/17646) ([Azat Khuzhin](https://github.com/azat)). +* Removed exception throwing at table initialization if there was no connection (it will be reconnecting in the background). [#17709](https://github.com/ClickHouse/ClickHouse/pull/17709) ([Kseniia Sumarokova](https://github.com/kssenii)). +* system.query_log now has extensive information to achieve better query analysis. [#17726](https://github.com/ClickHouse/ClickHouse/pull/17726) ([Amos Bird](https://github.com/amosbird)). +* Check system log tables' engine definition grammatically to prevent some configuration errors. Notes that this grammar check is not semantical, that means such mistakes as non-existent columns / expression functions would be not found out util the table is created. [#17739](https://github.com/ClickHouse/ClickHouse/pull/17739) ([Du Chuan](https://github.com/spongedu)). +* Improves the path concatenation of zookeeper paths inside DDLWorker. [#17767](https://github.com/ClickHouse/ClickHouse/pull/17767) ([Bharat Nallan](https://github.com/bharatnc)). +* Improvement of Web UI: do not add empty query to browser history. [#17770](https://github.com/ClickHouse/ClickHouse/pull/17770) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* - Let the possibility to override timeout value for running script using the ClickHouse docker image. [#17818](https://github.com/ClickHouse/ClickHouse/pull/17818) ([Guillaume Tassery](https://github.com/YiuRULE)). +* Add metrics(Parts, PartsActive, PartsInactive) for part number in MergeTree in clickhouse. [#17838](https://github.com/ClickHouse/ClickHouse/pull/17838) ([徐炘](https://github.com/weeds085490)). +* Add diagnostic information when two merge tables try to read each other's data. [#17854](https://github.com/ClickHouse/ClickHouse/pull/17854) ([徐炘](https://github.com/weeds085490)). +* Hints for column names. [#17112](https://github.com/ClickHouse/ClickHouse/issues/17112). [#17857](https://github.com/ClickHouse/ClickHouse/pull/17857) ([fastio](https://github.com/fastio)). +* Support for async tasks in `PipelineExecutor`. Initial support of async sockets for remote queries. [#17868](https://github.com/ClickHouse/ClickHouse/pull/17868) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* `allow_introspection_functions=0` prohibits usage of introspection functions but doesn't prohibit giving grants for them anymore (the grantee will need to set `allow_introspection_functions=1` for himself to be able to use that grant). Similarly `allow_ddl=0` prohibits usage of DDL commands but doesn't prohibit giving grants for them anymore. [#17908](https://github.com/ClickHouse/ClickHouse/pull/17908) ([Vitaly Baranov](https://github.com/vitlibar)). +* Ability to set custom metadata when putting S3 object. [#17909](https://github.com/ClickHouse/ClickHouse/pull/17909) ([Pavel Kovalenko](https://github.com/Jokser)). +* Adaptive choose of single/multi part upload in WriteBufferFromS3. Single part upload is controlled by a new setting 'max_single_part_upload_size'. [#17934](https://github.com/ClickHouse/ClickHouse/pull/17934) ([Pavel Kovalenko](https://github.com/Jokser)). +* Forcibly removing empty or bad metadata files from filesystem for DiskS3. S3 is an experimental feature. [#17935](https://github.com/ClickHouse/ClickHouse/pull/17935) ([Pavel Kovalenko](https://github.com/Jokser)). +* Now the table function `merge()` requires the current user to have the `SELECT` privilege on each table it receives data from. This PR fixes [#16964](https://github.com/ClickHouse/ClickHouse/issues/16964). [#17983](https://github.com/ClickHouse/ClickHouse/pull/17983) ([Vitaly Baranov](https://github.com/vitlibar)). +* Decrease log verbosity of the events when the client drops the connection from WARNING to INFORMATION. [#18005](https://github.com/ClickHouse/ClickHouse/pull/18005) ([filimonov](https://github.com/filimonov)). +* Fix clickhouse-client rendering issue when the size of terminal window changes. [#18009](https://github.com/ClickHouse/ClickHouse/pull/18009) ([Amos Bird](https://github.com/amosbird)). +* Temporary tables are visible in the system tables `system.tables` and `system.columns` now only in those session where they have been created. The internal database `_temporary_and_external_tables` is now hidden in those system tables; temporary tables are shown as tables with empty database with the `is_temporary` flag set instead. [#18014](https://github.com/ClickHouse/ClickHouse/pull/18014) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix dead list watches removal for TestKeeperStorage. [#18065](https://github.com/ClickHouse/ClickHouse/pull/18065) ([alesapin](https://github.com/alesapin)). +* Support `SHOW CREATE VIEW name` syntax like [MySQL](https://dev.mysql.com/doc/refman/5.7/en/show-create-view.html). [#18095](https://github.com/ClickHouse/ClickHouse/pull/18095) ([Du Chuan](https://github.com/spongedu)). +* Now the table function `merge()` requires the current user to have the `SELECT` privilege on each table it receives data from. This PR fixes [#16964](https://github.com/ClickHouse/ClickHouse/issues/16964). [#18104](https://github.com/ClickHouse/ClickHouse/pull/18104) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add `disk` to Set and Join storage settings. [#18112](https://github.com/ClickHouse/ClickHouse/pull/18112) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). +* `EXPLAIN AST` now support queries other then `SELECT`. [#18136](https://github.com/ClickHouse/ClickHouse/pull/18136) ([李扬](https://github.com/taiyang-li)). +* All queries of type `Decimal * Float` or vice versa are allowed, including aggregate ones (e.g. `SELECT sum(decimal_field * 1.1)` or `SELECT dec_col * float_col`), the result type is Float32 or Float64. [#18145](https://github.com/ClickHouse/ClickHouse/pull/18145) ([Mike Kot](https://github.com/myrrc)). +* Array aggregation `arrayMin`, `arrayMax`, `arraySum`, `arrayAvg` support for `Int128`, `Int256`, `UInt256`. [#18147](https://github.com/ClickHouse/ClickHouse/pull/18147) ([Maksim Kita](https://github.com/kitaisreal)). +* Better hints for `SHOW ...` query syntax. [#18183](https://github.com/ClickHouse/ClickHouse/pull/18183) ([Du Chuan](https://github.com/spongedu)). +* Now clickhouse-install could work on Mac. The problem was that there is no procfs on this platform. [#18201](https://github.com/ClickHouse/ClickHouse/pull/18201) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Make better adaptive granularity calculation for merge tree wide parts. [#18223](https://github.com/ClickHouse/ClickHouse/pull/18223) ([alesapin](https://github.com/alesapin)). +* Allow to parse Array fields from CSV if it is represented as a string containing array that was serialized as nested CSV. Example: `"[""Hello"", ""world"", ""42"""" TV""]"` will parse as `['Hello', 'world', '42" TV']`. Allow to parse array in CSV in a string without enclosing braces. Example: `"'Hello', 'world', '42"" TV'"` will parse as `['Hello', 'world', '42" TV']`. [#18271](https://github.com/ClickHouse/ClickHouse/pull/18271) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* In case of unexpected exceptions automatically restart background thread which is responsible for execution of distributed DDL queries. Fixes [#17991](https://github.com/ClickHouse/ClickHouse/issues/17991). [#18285](https://github.com/ClickHouse/ClickHouse/pull/18285) ([徐炘](https://github.com/weeds085490)). +* Add a new setting `insert_distributed_one_random_shard = 1` to allow insertion into multi-sharded distributed table without any distributed key. [#18294](https://github.com/ClickHouse/ClickHouse/pull/18294) ([Amos Bird](https://github.com/amosbird)). +* related to [#18133](https://github.com/ClickHouse/ClickHouse/issues/18133). [#18309](https://github.com/ClickHouse/ClickHouse/pull/18309) ([hexiaoting](https://github.com/hexiaoting)). +* Fix potential server crash during Buffer rollback (that is impossible in current ClickHouse version). [#18329](https://github.com/ClickHouse/ClickHouse/pull/18329) ([Azat Khuzhin](https://github.com/azat)). +* Support builtin function `isIPv4String` && `isIPv6String` like [MySQL](https://github.com/ClickHouse/ClickHouse/compare/master...spongedu:support_is_ipv4?expand=1). [#18349](https://github.com/ClickHouse/ClickHouse/pull/18349) ([Du Chuan](https://github.com/spongedu)). +* Add ability to modify primary and partition key column type from `LowCardinality(Type)` to `Type` and vice versa. Also add an ability to modify primary key column type from `EnumX ` to `IntX` type. Fixes [#5604](https://github.com/ClickHouse/ClickHouse/issues/5604). [#18362](https://github.com/ClickHouse/ClickHouse/pull/18362) ([alesapin](https://github.com/alesapin)). +* Fix bug: no newline after exception message in some tools. [#18444](https://github.com/ClickHouse/ClickHouse/pull/18444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* support syntax `EXISTS DATABASE name`. [#18458](https://github.com/ClickHouse/ClickHouse/pull/18458) ([Du Chuan](https://github.com/spongedu)). +* Fixed assertion error inside allocator in case when last argument of function bar is NaN. Now simple ClickHouse's exception is being thrown. This fixes [#17876](https://github.com/ClickHouse/ClickHouse/issues/17876). [#18520](https://github.com/ClickHouse/ClickHouse/pull/18520) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* PODArray: Avoid call to memcpy with (nullptr, 0) arguments (Fix UBSan report). This fixes [#18525](https://github.com/ClickHouse/ClickHouse/issues/18525). [#18526](https://github.com/ClickHouse/ClickHouse/pull/18526) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix SimpleAggregateFunction in SummingMergeTree. Now it works like AggregateFunction. In previous versions values were summed together regardless to the aggregate function. This fixes [#18564](https://github.com/ClickHouse/ClickHouse/issues/18564) . [#8052](https://github.com/ClickHouse/ClickHouse/issues/8052). [#18637](https://github.com/ClickHouse/ClickHouse/pull/18637) ([Amos Bird](https://github.com/amosbird)). +* Another fix of using SimpleAggregateFunction in SummingMergeTree. This fixes [#18676](https://github.com/ClickHouse/ClickHouse/issues/18676) . [#18677](https://github.com/ClickHouse/ClickHouse/pull/18677) ([Amos Bird](https://github.com/amosbird)). +* Allow column transformer `EXCEPT` to accept a string as regular expression matcher. This resolves [#18685](https://github.com/ClickHouse/ClickHouse/issues/18685) . [#18699](https://github.com/ClickHouse/ClickHouse/pull/18699) ([Amos Bird](https://github.com/amosbird)). +* Apply `ALTER TABLE ON CLUSTER MODIFY SETTING ...` to all replicas. Because we don't replicate such alter commands. [#18789](https://github.com/ClickHouse/ClickHouse/pull/18789) ([Amos Bird](https://github.com/amosbird)). +* Expand macros in the zk path when executing fetchPartition. [#18839](https://github.com/ClickHouse/ClickHouse/pull/18839) ([fastio](https://github.com/fastio)). +* `SYSTEM KILL` command started to work in Docker. This closes [#18847](https://github.com/ClickHouse/ClickHouse/issues/18847). [#18848](https://github.com/ClickHouse/ClickHouse/pull/18848) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Two new settings (by analogy with MergeTree family) has been added: - `fsync_after_insert` - Do fsync for every inserted. Will decreases performance of inserts. - `fsync_directories` - Do fsync for temporary directory (that is used for async INSERT only) after all operations (writes, renames, etc.). [#18864](https://github.com/ClickHouse/ClickHouse/pull/18864) ([Azat Khuzhin](https://github.com/azat)). +* change the sorting key of events_list from timestamp to (timestamp, event_index). [#18884](https://github.com/ClickHouse/ClickHouse/pull/18884) ([Fuwang Hu](https://github.com/fuwhu)). +* Aliases declared in `WITH` statement are properly used in index analysis. Queries like `WITH column AS alias SELECT ... WHERE alias = ...` may use index now. [#18896](https://github.com/ClickHouse/ClickHouse/pull/18896) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix +* Fix bug when clickhouse-server doesn't send `close` request to ZooKeeper server. [#16837](https://github.com/ClickHouse/ClickHouse/pull/16837) ([alesapin](https://github.com/alesapin)). +* TODO. [#16866](https://github.com/ClickHouse/ClickHouse/pull/16866) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix optimization of group by with enabled setting `optimize_aggregators_of_group_by_keys` and joins. Fixes [#12604](https://github.com/ClickHouse/ClickHouse/issues/12604). [#16951](https://github.com/ClickHouse/ClickHouse/pull/16951) ([Anton Popov](https://github.com/CurtizJ)). +* Fix incorrect comparison of types `DateTime64` with different scales. Fixes [#16655](https://github.com/ClickHouse/ClickHouse/issues/16655) ... [#16952](https://github.com/ClickHouse/ClickHouse/pull/16952) ([Vasily Nemkov](https://github.com/Enmk)). +* Fix order by optimization with monotonous functions. Fixes [#16107](https://github.com/ClickHouse/ClickHouse/issues/16107). [#16956](https://github.com/ClickHouse/ClickHouse/pull/16956) ([Anton Popov](https://github.com/CurtizJ)). +* Fix Merge(Distributed()) with JOIN. [#16993](https://github.com/ClickHouse/ClickHouse/pull/16993) ([Azat Khuzhin](https://github.com/azat)). +* - Fix optimize_distributed_group_by_sharding_key for query with OFFSET only. [#16996](https://github.com/ClickHouse/ClickHouse/pull/16996) ([Azat Khuzhin](https://github.com/azat)). +* Bug fix for funciton fuzzBits, related issue: [#16980](https://github.com/ClickHouse/ClickHouse/issues/16980). [#17051](https://github.com/ClickHouse/ClickHouse/pull/17051) ([hexiaoting](https://github.com/hexiaoting)). +* Fix possible wrong index analysis when the types of the index comparison are different. This fixes [#17122](https://github.com/ClickHouse/ClickHouse/issues/17122). [#17145](https://github.com/ClickHouse/ClickHouse/pull/17145) ([Amos Bird](https://github.com/amosbird)). +* Fixed possible not-working mutations for parts stored on S3 disk. [#17227](https://github.com/ClickHouse/ClickHouse/pull/17227) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix [#15235](https://github.com/ClickHouse/ClickHouse/issues/15235). When clickhouse-copier handle non-partitioned table, throws segfault error. [#17248](https://github.com/ClickHouse/ClickHouse/pull/17248) ([Qi Chen](https://github.com/kaka11chen)). +* Fix set index invalidation when there are const columns in the subquery. This fixes [#17246](https://github.com/ClickHouse/ClickHouse/issues/17246) . [#17249](https://github.com/ClickHouse/ClickHouse/pull/17249) ([Amos Bird](https://github.com/amosbird)). +* Fix possible `Unexpected packet Data received from client` error for Distributed queries with `LIMIT`. [#17254](https://github.com/ClickHouse/ClickHouse/pull/17254) ([Azat Khuzhin](https://github.com/azat)). +* Fix indeterministic functions with predicate optimizer. This fixes [#17244](https://github.com/ClickHouse/ClickHouse/issues/17244). [#17273](https://github.com/ClickHouse/ClickHouse/pull/17273) ([Winter Zhang](https://github.com/zhang2014)). +* fixes [#16835](https://github.com/ClickHouse/ClickHouse/issues/16835) try fix miss match header with MySQL SHOW statement. [#17366](https://github.com/ClickHouse/ClickHouse/pull/17366) ([Winter Zhang](https://github.com/zhang2014)). +* Fix crash while reading from `JOIN` table with `LowCardinality` types. Fixes [#17228](https://github.com/ClickHouse/ClickHouse/issues/17228). [#17397](https://github.com/ClickHouse/ClickHouse/pull/17397) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed high CPU usage in background tasks of *MergeTree tables. [#17416](https://github.com/ClickHouse/ClickHouse/pull/17416) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix duplicates after `DISTINCT` which were possible because of incorrect optimization. Fixes [#17294](https://github.com/ClickHouse/ClickHouse/issues/17294). [#17296](https://github.com/ClickHouse/ClickHouse/pull/17296) ([li chengxiang](https://github.com/chengxianglibra)). [#17439](https://github.com/ClickHouse/ClickHouse/pull/17439) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Avoid server abnormal termination in case of too low memory limits (`max_memory_usage=1`/`max_untracked_memory=1`). [#17453](https://github.com/ClickHouse/ClickHouse/pull/17453) ([Azat Khuzhin](https://github.com/azat)). +* Fix `ORDER BY` with enabled setting `optimize_redundant_functions_in_order_by`. [#17471](https://github.com/ClickHouse/ClickHouse/pull/17471) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bug when mark cache size was underestimated by clickhouse. It may happen when there are a lot of tiny files with marks. [#17496](https://github.com/ClickHouse/ClickHouse/pull/17496) ([alesapin](https://github.com/alesapin)). +* Fix alter query hang when the corresponding mutation was killed on the different replica. Fixes [#16953](https://github.com/ClickHouse/ClickHouse/issues/16953). [#17499](https://github.com/ClickHouse/ClickHouse/pull/17499) ([alesapin](https://github.com/alesapin)). +* Fix the issue when server can stop accepting connections in very rare cases. [#17542](https://github.com/ClickHouse/ClickHouse/pull/17542) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* When clickhouse-client is used in interactive mode with multiline queries, single line comment was erronously extended till the end of query. This fixes [#13654](https://github.com/ClickHouse/ClickHouse/issues/13654). [#17565](https://github.com/ClickHouse/ClickHouse/pull/17565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Exception `fmt::v7::format_error` can be logged in background for MergeTree tables. This fixes [#17613](https://github.com/ClickHouse/ClickHouse/issues/17613). [#17615](https://github.com/ClickHouse/ClickHouse/pull/17615) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix empty `system.stack_trace` table when server is running in daemon mode. [#17630](https://github.com/ClickHouse/ClickHouse/pull/17630) ([Amos Bird](https://github.com/amosbird)). +* In might be determined incorrectly if cluster is circular- (cross-) replicated or not when executing `ON CLUSTER` query due to race condition when `pool_size` > 1. It's fixed. [#17640](https://github.com/ClickHouse/ClickHouse/pull/17640) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed `Function not implemented` error when executing `RENAME` query in `Atomic` database with ClickHouse running on Windows Subsystem for Linux. Fixes [#17661](https://github.com/ClickHouse/ClickHouse/issues/17661). [#17664](https://github.com/ClickHouse/ClickHouse/pull/17664) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed problem when ClickHouse fails to resume connection to MySQL servers. [#17681](https://github.com/ClickHouse/ClickHouse/pull/17681) ([Alexander Kazakov](https://github.com/Akazz)). +* Fixed segfault when there is not enough space when inserting into `Distributed` table. [#17737](https://github.com/ClickHouse/ClickHouse/pull/17737) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Exception message about max table size to drop was displayed incorrectly. [#17764](https://github.com/ClickHouse/ClickHouse/pull/17764) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not restore parts from WAL if `in_memory_parts_enable_wal` is disabled. [#17802](https://github.com/ClickHouse/ClickHouse/pull/17802) ([detailyang](https://github.com/detailyang)). +* fix incorrect initialize `max_compress_block_size` of MergeTreeWriterSettings with `min_compress_block_size`. [#17833](https://github.com/ClickHouse/ClickHouse/pull/17833) ([flynn](https://github.com/ucasfl)). +* Fix possible segfault in `topK` aggregate function. This closes [#17404](https://github.com/ClickHouse/ClickHouse/issues/17404). [#17845](https://github.com/ClickHouse/ClickHouse/pull/17845) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix max_distributed_connections (affects `prefer_localhost_replica=1` and `max_threads!=max_distributed_connections`). [#17848](https://github.com/ClickHouse/ClickHouse/pull/17848) ([Azat Khuzhin](https://github.com/azat)). +* Trivial query optimization was producing wrong result if query contains ARRAY JOIN (so query is actually non trivial). [#17887](https://github.com/ClickHouse/ClickHouse/pull/17887) ([sundyli](https://github.com/sundy-li)). +* Fix comparison of `DateTime64` and `Date`. Fixes [#13804](https://github.com/ClickHouse/ClickHouse/issues/13804) and [#11222](https://github.com/ClickHouse/ClickHouse/issues/11222). ... [#17895](https://github.com/ClickHouse/ClickHouse/pull/17895) ([Vasily Nemkov](https://github.com/Enmk)). +* When server log rotation was configured using `logger.size` parameter with numeric value larger than 2^32, the logs were not rotated properly. This is fixed. [#17905](https://github.com/ClickHouse/ClickHouse/pull/17905) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* fixes [#15187](https://github.com/ClickHouse/ClickHouse/issues/15187) fixes [#17912](https://github.com/ClickHouse/ClickHouse/issues/17912) support convert MySQL prefix index for MaterializeMySQL CC: @tavplubix. [#17944](https://github.com/ClickHouse/ClickHouse/pull/17944) ([Winter Zhang](https://github.com/zhang2014)). +* Fix comparison of `DateTime64` and `Date`. Fixes [#13804](https://github.com/ClickHouse/ClickHouse/issues/13804) and [#11222](https://github.com/ClickHouse/ClickHouse/issues/11222). ... [#18050](https://github.com/ClickHouse/ClickHouse/pull/18050) ([Vasily Nemkov](https://github.com/Enmk)). +* Fix corruption in librdkafka snappy decompression (was a problem only for gcc10 builds, but official builds uses clang already, so at least recent official releases are not affected). [#18053](https://github.com/ClickHouse/ClickHouse/pull/18053) ([Azat Khuzhin](https://github.com/azat)). +* Fixed `std::out_of_range: basic_string` in S3 URL parsing. [#18059](https://github.com/ClickHouse/ClickHouse/pull/18059) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix error when query `MODIFY COLUMN ... REMOVE TTL` doesn't actually remove column TTL. [#18130](https://github.com/ClickHouse/ClickHouse/pull/18130) ([alesapin](https://github.com/alesapin)). +* Fix `Unknown setting profile` error on attempt to set settings profile. [#18167](https://github.com/ClickHouse/ClickHouse/pull/18167) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix inserting a row with default value in case of parsing error in the last column. Fixes [#17712](https://github.com/ClickHouse/ClickHouse/issues/17712). [#18182](https://github.com/ClickHouse/ClickHouse/pull/18182) ([Jianmei Zhang](https://github.com/zhangjmruc)). +* fixes [#18186](https://github.com/ClickHouse/ClickHouse/issues/18186) fixes [#16372](https://github.com/ClickHouse/ClickHouse/issues/16372) fix unique key convert crash in MaterializeMySQL database engine. [#18211](https://github.com/ClickHouse/ClickHouse/pull/18211) ([Winter Zhang](https://github.com/zhang2014)). +* Fix key comparison between Enum and Int types. This fixes [#17989](https://github.com/ClickHouse/ClickHouse/issues/17989). [#18214](https://github.com/ClickHouse/ClickHouse/pull/18214) ([Amos Bird](https://github.com/amosbird)). +* Fix possible incomplete query result while reading from `MergeTree*` in case of read backoff (message ` MergeTreeReadPool: Will lower number of threads` in logs). Was introduced in [#16423](https://github.com/ClickHouse/ClickHouse/issues/16423). Fixes [#18137](https://github.com/ClickHouse/ClickHouse/issues/18137). [#18216](https://github.com/ClickHouse/ClickHouse/pull/18216) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* `SELECT JOIN` now requires the `SELECT` privilege on each of the joined tables. This PR fixes [#17654](https://github.com/ClickHouse/ClickHouse/issues/17654). [#18232](https://github.com/ClickHouse/ClickHouse/pull/18232) ([Vitaly Baranov](https://github.com/vitlibar)). +* - Fixed issue when `clickhouse-odbc-bridge` process is unreachable by server on machines with dual IPv4/IPv6 stack; - Fixed issue when ODBC dictionary updates are performed using malformed queries and/or cause crashes; Possibly closes [#14489](https://github.com/ClickHouse/ClickHouse/issues/14489). [#18278](https://github.com/ClickHouse/ClickHouse/pull/18278) ([Denis Glazachev](https://github.com/traceon)). +* Fix possible crashes in aggregate functions with combinator `Distinct`, while using two-level aggregation. Fixes [#17682](https://github.com/ClickHouse/ClickHouse/issues/17682). [#18365](https://github.com/ClickHouse/ClickHouse/pull/18365) ([Anton Popov](https://github.com/CurtizJ)). +* Fix filling table `system.settings_profile_elements`. This PR fixes [#18231](https://github.com/ClickHouse/ClickHouse/issues/18231). [#18379](https://github.com/ClickHouse/ClickHouse/pull/18379) ([Vitaly Baranov](https://github.com/vitlibar)). +* Restrict merges from wide to compact parts. In case of vertical merge it led to broken result part. [#18381](https://github.com/ClickHouse/ClickHouse/pull/18381) ([Anton Popov](https://github.com/CurtizJ)). +* Fix possible race condition in concurrent usage of `Set` or `Join` tables and selects from `system.tables`. [#18385](https://github.com/ClickHouse/ClickHouse/pull/18385) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix -SimpleState combinator generates incompatible arugment type and return type. [#18404](https://github.com/ClickHouse/ClickHouse/pull/18404) ([Amos Bird](https://github.com/amosbird)). +* Fix the unexpected behaviour of `SHOW TABLES`. [#18431](https://github.com/ClickHouse/ClickHouse/pull/18431) ([fastio](https://github.com/fastio)). +* Fixed `value is too short` error when executing `toType(...)` functions (`toDate`, `toUInt32`, etc) with argument of type `Nullable(String)`. Now such functions return `NULL` on parsing errors instead of throwing exception. Fixes [#7673](https://github.com/ClickHouse/ClickHouse/issues/7673). [#18445](https://github.com/ClickHouse/ClickHouse/pull/18445) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Proper support for 12AM in `parseDateTimeBestEffort` function. This fixes [#18402](https://github.com/ClickHouse/ClickHouse/issues/18402). [#18449](https://github.com/ClickHouse/ClickHouse/pull/18449) ([vladimir-golovchenko](https://github.com/vladimir-golovchenko)). +* Disable write with AIO during merges because it can lead to extremely rare data corruption of primary key columns during merge. [#18481](https://github.com/ClickHouse/ClickHouse/pull/18481) ([alesapin](https://github.com/alesapin)). +* Fix bug which may lead to `ALTER` queries hung after corresponding mutation kill. Found by thread fuzzer. [#18518](https://github.com/ClickHouse/ClickHouse/pull/18518) ([alesapin](https://github.com/alesapin)). +* Fix possible `Pipeline stuck` error while using `ORDER BY` after subquery with `RIGHT` or `FULL` join. [#18550](https://github.com/ClickHouse/ClickHouse/pull/18550) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add FixedString Data type support. I'll get this exception "Code: 50, e.displayText() = DB::Exception: Unsupported type FixedString(1)" when replicating data from MySQL to ClickHouse. This patch fixes bug [#18450](https://github.com/ClickHouse/ClickHouse/issues/18450) Also fixes [#6556](https://github.com/ClickHouse/ClickHouse/issues/6556). [#18553](https://github.com/ClickHouse/ClickHouse/pull/18553) ([awesomeleo](https://github.com/awesomeleo)). +* Fix previous bug when date overflow with different values. Strict Date value limit to "2106-02-07", cast date > "2106-02-07" to value 0. [#18565](https://github.com/ClickHouse/ClickHouse/pull/18565) ([hexiaoting](https://github.com/hexiaoting)). +* Fix removing of empty parts in `ReplicatedMergeTree` tables, created with old syntax. Fixes [#18582](https://github.com/ClickHouse/ClickHouse/issues/18582). [#18614](https://github.com/ClickHouse/ClickHouse/pull/18614) ([Anton Popov](https://github.com/CurtizJ)). +* Fix Logger with unmatched arg size. [#18717](https://github.com/ClickHouse/ClickHouse/pull/18717) ([sundyli](https://github.com/sundy-li)). +* Fixed `Attempt to read after eof` error when trying to `CAST` `NULL` from `Nullable(String)` to `Nullable(Decimal(P, S))`. Now function `CAST` returns `NULL` when it cannot parse decimal from nullable string. Fixes [#7690](https://github.com/ClickHouse/ClickHouse/issues/7690). [#18718](https://github.com/ClickHouse/ClickHouse/pull/18718) ([Winter Zhang](https://github.com/zhang2014)). +* Asynchronous distributed INSERTs can be rejected by the server if the setting `network_compression_method` is globally set to non-default value. This fixes [#18741](https://github.com/ClickHouse/ClickHouse/issues/18741). [#18776](https://github.com/ClickHouse/ClickHouse/pull/18776) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix *If combinator with unary function and Nullable types. [#18806](https://github.com/ClickHouse/ClickHouse/pull/18806) ([Azat Khuzhin](https://github.com/azat)). +* - Fix never worked `fsync_part_directory`/`fsync_after_insert`/`in_memory_parts_insert_sync`. [#18845](https://github.com/ClickHouse/ClickHouse/pull/18845) ([Azat Khuzhin](https://github.com/azat)). +* Fix use after free bug in rocksdb. [#18862](https://github.com/ClickHouse/ClickHouse/pull/18862) ([sundyli](https://github.com/sundy-li)). +* Queries for external databases (MySQL, ODBC, JDBC) were incorrectly rewritten if there was an expression in form of `x IN table`. This fixes [#9756](https://github.com/ClickHouse/ClickHouse/issues/9756). [#18876](https://github.com/ClickHouse/ClickHouse/pull/18876) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix possible hang at shutdown in clickhouse-local. This fixes [#18891](https://github.com/ClickHouse/ClickHouse/issues/18891). [#18893](https://github.com/ClickHouse/ClickHouse/pull/18893) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix issue with `bitmapOrCardinality` that may lead to nullptr dereference. This closes [#18911](https://github.com/ClickHouse/ClickHouse/issues/18911). [#18912](https://github.com/ClickHouse/ClickHouse/pull/18912) ([sundyli](https://github.com/sundy-li)). + +#### Build/Testing/Packaging Improvement +* Add some test for MaterializeMySQL. e.g. network partition, MySQL kill sync thread... [#16806](https://github.com/ClickHouse/ClickHouse/pull/16806) ([TCeason](https://github.com/TCeason)). +* Now ClickHouse can pretend to be a fake ZooKeeper. Currently, storage implementation is just stored in-memory hash-table, and server partially support ZooKeeper protocol. [#16877](https://github.com/ClickHouse/ClickHouse/pull/16877) ([alesapin](https://github.com/alesapin)). +* * Added RBAC tests for `ATTACH`, `CREATE`, `DROP`, and `DETACH`. [#16977](https://github.com/ClickHouse/ClickHouse/pull/16977) ([MyroTk](https://github.com/MyroTk)). +* `PODArray` does not initialize "new" elements when resizing, unlike `std::vector`. This probably fixes [this failure](https://clickhouse-test-reports.s3.yandex.net/17309/065cd002578f2e8228f12a2744bd40c970065e0c/stress_test_(memory)/stderr.log) from [#17309](https://github.com/ClickHouse/ClickHouse/issues/17309). [#17344](https://github.com/ClickHouse/ClickHouse/pull/17344) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* There was an uninitialized variable in the code of Copier. [#17363](https://github.com/ClickHouse/ClickHouse/pull/17363) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Send info about official build, memory, cpu and free disk space to Sentry if it is enabled. Sentry is opt-in feature to help ClickHouse developers. This closes [#17279](https://github.com/ClickHouse/ClickHouse/issues/17279). [#17543](https://github.com/ClickHouse/ClickHouse/pull/17543) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add an integration test: MySQL server killed while insert for MaterializeMySQL Engine. [#17614](https://github.com/ClickHouse/ClickHouse/pull/17614) ([TCeason](https://github.com/TCeason)). +* Add an integration test: ClickHouse killed while insert for MaterializeMySQL ENGINE. [#17622](https://github.com/ClickHouse/ClickHouse/pull/17622) ([TCeason](https://github.com/TCeason)). +* - RBAC testflows tests for SHOW, TRUNCATE, KILL, and OPTIMIZE. - Updates to old tests. - Resolved comments from #https://github.com/ClickHouse/ClickHouse/pull/16977. [#17657](https://github.com/ClickHouse/ClickHouse/pull/17657) ([MyroTk](https://github.com/MyroTk)). +* Now we use the fresh docker daemon version in integration tests. [#17671](https://github.com/ClickHouse/ClickHouse/pull/17671) ([alesapin](https://github.com/alesapin)). +* - Testflows tests for RBAC [ACCESS MANAGEMENT](https://clickhouse.tech/docs/en/sql-reference/statements/grant/#grant-access-management) privileges. [#17804](https://github.com/ClickHouse/ClickHouse/pull/17804) ([MyroTk](https://github.com/MyroTk)). +* Updating TestFlows README.md to include "How To Debug Why Test Failed" section. [#17808](https://github.com/ClickHouse/ClickHouse/pull/17808) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add our own CMakeList for dragonbox which was added in [#17831](https://github.com/ClickHouse/ClickHouse/issues/17831). [#17869](https://github.com/ClickHouse/ClickHouse/pull/17869) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Enable Pytest framework for stateless tests. [#17902](https://github.com/ClickHouse/ClickHouse/pull/17902) ([Ivan](https://github.com/abyss7)). +* Adjusting timeouts a bit, in the good hope that it will prevent flakiness of the test. [#18000](https://github.com/ClickHouse/ClickHouse/pull/18000) ([filimonov](https://github.com/filimonov)). +* Now, `clickhouse-test` DROP/CREATE databases with a timeout. [#18098](https://github.com/ClickHouse/ClickHouse/pull/18098) ([alesapin](https://github.com/alesapin)). +* Change OpenSSL to BoringSSL. It allows to avoid issues with sanitizers. This fixes [#12490](https://github.com/ClickHouse/ClickHouse/issues/12490). This fixes [#17502](https://github.com/ClickHouse/ClickHouse/issues/17502). This fixes [#12952](https://github.com/ClickHouse/ClickHouse/issues/12952). [#18129](https://github.com/ClickHouse/ClickHouse/pull/18129) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix usage of uninitialized value in function toModifiedJulianDayOrNull, reported by MSan. Was discovered [here](https://github.com/ClickHouse/ClickHouse/pull/17726#issuecomment-744050500). [#18172](https://github.com/ClickHouse/ClickHouse/pull/18172) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update `anchore/scan-action@main` workflow action (was moved from master). [#18192](https://github.com/ClickHouse/ClickHouse/pull/18192) ([Stig Bakken](https://github.com/stigsb)). +* Do not use non thread-safe function `strerror`. [#18204](https://github.com/ClickHouse/ClickHouse/pull/18204) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* - Updating TestFlows version to the latest 1.6.72 - Re-generating requirements.py. [#18208](https://github.com/ClickHouse/ClickHouse/pull/18208) ([vzakaznikov](https://github.com/vzakaznikov)). +* Merging requirements for AES encryption functions. Updating aes_encryption tests to use new requirements. Updating TestFlows version to 1.6.72. [#18221](https://github.com/ClickHouse/ClickHouse/pull/18221) ([vzakaznikov](https://github.com/vzakaznikov)). +* Enable Thread Fuzzer for stateless tests flaky check. [#18299](https://github.com/ClickHouse/ClickHouse/pull/18299) ([alesapin](https://github.com/alesapin)). +* Check for leftovers of conflict markers in docs. [#18332](https://github.com/ClickHouse/ClickHouse/pull/18332) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix codespell warnings. Split style checks into separate parts. Update style checks docker image. [#18463](https://github.com/ClickHouse/ClickHouse/pull/18463) ([Ilya Yatsishin](https://github.com/qoega)). +* Update timezones info to 2020e. [#18531](https://github.com/ClickHouse/ClickHouse/pull/18531) ([alesapin](https://github.com/alesapin)). +* Fix shellcheck errors in style check. [#18566](https://github.com/ClickHouse/ClickHouse/pull/18566) ([Ilya Yatsishin](https://github.com/qoega)). +* TestFlows: fixes to LDAP tests that fail due to slow test execution. [#18790](https://github.com/ClickHouse/ClickHouse/pull/18790) ([vzakaznikov](https://github.com/vzakaznikov)). +* Generate build id when ClickHouse is linked with `lld`. It's appeared that `lld` does not generate it by default on my machine. Build id is used for crash reports and introspection. [#18808](https://github.com/ClickHouse/ClickHouse/pull/18808) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add simple integrity check for ClickHouse binary. It allows to detect corruption due to faulty hardware (bit rot on storage media or bit flips in RAM). [#18811](https://github.com/ClickHouse/ClickHouse/pull/18811) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Removed the -finline-hint-functions flag not present in GCC. [#18846](https://github.com/ClickHouse/ClickHouse/pull/18846) ([Mike Kot](https://github.com/myrrc)). +* Add `SYSTEM SUSPEND` command for fault injection. It can be used to faciliate failover tests. This closes [#15979](https://github.com/ClickHouse/ClickHouse/issues/15979). [#18850](https://github.com/ClickHouse/ClickHouse/pull/18850) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Bump mkdocs-macros-plugin from 0.4.20 to 0.5.0 in /docs/tools'. [#17351](https://github.com/ClickHouse/ClickHouse/pull/17351) ([dependabot-preview[bot]](https://github.com/apps/dependabot-preview)). +* NO CL ENTRY: 'Revert "Attempt to fix Stress test (MSan)"'. [#17372](https://github.com/ClickHouse/ClickHouse/pull/17372) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* NO CL ENTRY: 'Revert "Bump mkdocs-macros-plugin from 0.4.20 to 0.5.0 in /docs/tools"'. [#17405](https://github.com/ClickHouse/ClickHouse/pull/17405) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Update README.md'. [#17596](https://github.com/ClickHouse/ClickHouse/pull/17596) ([Robert Hodges](https://github.com/hodgesrm)). +* NO CL ENTRY: 'Revert "Fix index granularity calculation on block borders"'. [#17918](https://github.com/ClickHouse/ClickHouse/pull/17918) ([alesapin](https://github.com/alesapin)). +* NO CL ENTRY: 'Revert "Date vs DateTime64 comparison"'. [#17985](https://github.com/ClickHouse/ClickHouse/pull/17985) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Исправил опечатку в названии ОС RedHad->RedHat'. [#18028](https://github.com/ClickHouse/ClickHouse/pull/18028) ([Ed Rakhmankulov](https://github.com/Erixonich)). +* NO CL ENTRY: 'Revert "Fix access rights required for the merge() table function."'. [#18103](https://github.com/ClickHouse/ClickHouse/pull/18103) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Add some extra tests to copier"'. [#18636](https://github.com/ClickHouse/ClickHouse/pull/18636) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* NO CL ENTRY: 'Fix typo in array functions' documentation'. [#18792](https://github.com/ClickHouse/ClickHouse/pull/18792) ([Bertrand Junqua](https://github.com/Bertrand31)). +* NO CL ENTRY: 'Revert "Add metrics for part number in MergeTree in ClickHouse"'. [#18834](https://github.com/ClickHouse/ClickHouse/pull/18834) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Fixed typo in metrics.md'. [#18920](https://github.com/ClickHouse/ClickHouse/pull/18920) ([Mark Frost](https://github.com/frostmark)). + diff --git a/docs/changelogs/v21.1.2.15-stable.md b/docs/changelogs/v21.1.2.15-stable.md new file mode 100644 index 00000000000..205794b94c2 --- /dev/null +++ b/docs/changelogs/v21.1.2.15-stable.md @@ -0,0 +1,28 @@ +### ClickHouse release v21.1.2.15-stable FIXME as compared to v21.1.1.5646-prestable + +#### Improvement +* Backported in [#19148](https://github.com/ClickHouse/ClickHouse/issues/19148): Explicitly set uid / gid of clickhouse user & group to the fixed values (101) in clickhouse-server images. [#19096](https://github.com/ClickHouse/ClickHouse/pull/19096) ([filimonov](https://github.com/filimonov)). + +#### Bug Fix +* Backported in [#19203](https://github.com/ClickHouse/ClickHouse/issues/19203): `SELECT count() FROM table` now can be executed if only one any column can be selected from the `table`. This PR fixes [#10639](https://github.com/ClickHouse/ClickHouse/issues/10639). [#18233](https://github.com/ClickHouse/ClickHouse/pull/18233) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#19161](https://github.com/ClickHouse/ClickHouse/issues/19161): Fix index analysis of binary functions with constant argument which leads to wrong query results. This fixes [#18364](https://github.com/ClickHouse/ClickHouse/issues/18364). [#18373](https://github.com/ClickHouse/ClickHouse/pull/18373) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#18948](https://github.com/ClickHouse/ClickHouse/issues/18948): Fixed `Attempt to read after eof` error when trying to `CAST` `NULL` from `Nullable(String)` to `Nullable(Decimal(P, S))`. Now function `CAST` returns `NULL` when it cannot parse decimal from nullable string. Fixes [#7690](https://github.com/ClickHouse/ClickHouse/issues/7690). [#18718](https://github.com/ClickHouse/ClickHouse/pull/18718) ([Winter Zhang](https://github.com/zhang2014)). +* Backported in [#18934](https://github.com/ClickHouse/ClickHouse/issues/18934): Fix issue with `bitmapOrCardinality` that may lead to nullptr dereference. This closes [#18911](https://github.com/ClickHouse/ClickHouse/issues/18911). [#18912](https://github.com/ClickHouse/ClickHouse/pull/18912) ([sundyli](https://github.com/sundy-li)). +* Backported in [#19115](https://github.com/ClickHouse/ClickHouse/issues/19115): Attach partition should reset the mutation. [#18804](https://github.com/ClickHouse/ClickHouse/issues/18804). [#18935](https://github.com/ClickHouse/ClickHouse/pull/18935) ([fastio](https://github.com/fastio)). +* Backported in [#18966](https://github.com/ClickHouse/ClickHouse/issues/18966): Fix bug when mutation with some escaped text (like `ALTER ... UPDATE e = CAST('foo', 'Enum8(\'foo\' = 1')` serialized incorrectly. Fixes [#18878](https://github.com/ClickHouse/ClickHouse/issues/18878). [#18944](https://github.com/ClickHouse/ClickHouse/pull/18944) ([alesapin](https://github.com/alesapin)). +* Backported in [#19005](https://github.com/ClickHouse/ClickHouse/issues/19005): Fix error `Task was not found in task queue` (possible only for remote queries, with `async_socket_for_remote = 1`). [#18964](https://github.com/ClickHouse/ClickHouse/pull/18964) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19009](https://github.com/ClickHouse/ClickHouse/issues/19009): Fix incorrect behavior when `ALTER TABLE ... DROP PART 'part_name'` query removes all deduplication blocks for the whole partition. Fixes [#18874](https://github.com/ClickHouse/ClickHouse/issues/18874). [#18969](https://github.com/ClickHouse/ClickHouse/pull/18969) ([alesapin](https://github.com/alesapin)). +* Backported in [#19193](https://github.com/ClickHouse/ClickHouse/issues/19193): Fixed very rare deadlock at shutdown. [#18977](https://github.com/ClickHouse/ClickHouse/pull/18977) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19004](https://github.com/ClickHouse/ClickHouse/issues/19004): Fix possible exception `QueryPipeline stream: different number of columns` caused by merging of query plan's `Expression` steps. Fixes [#18190](https://github.com/ClickHouse/ClickHouse/issues/18190). [#18980](https://github.com/ClickHouse/ClickHouse/pull/18980) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19045](https://github.com/ClickHouse/ClickHouse/issues/19045): Disable `optimize_move_functions_out_of_any` because optimization is not always correct. This closes [#18051](https://github.com/ClickHouse/ClickHouse/issues/18051). This closes [#18973](https://github.com/ClickHouse/ClickHouse/issues/18973). [#18981](https://github.com/ClickHouse/ClickHouse/pull/18981) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19070](https://github.com/ClickHouse/ClickHouse/issues/19070): Join tries to materialize const columns, but our code waits for them in other places. [#18982](https://github.com/ClickHouse/ClickHouse/pull/18982) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#19053](https://github.com/ClickHouse/ClickHouse/issues/19053): Fix inserting of `LowCardinality` column to table with `TinyLog` engine. Fixes [#18629](https://github.com/ClickHouse/ClickHouse/issues/18629). [#19010](https://github.com/ClickHouse/ClickHouse/pull/19010) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19081](https://github.com/ClickHouse/ClickHouse/issues/19081): Fix possible error `Expected single dictionary argument for function` if use function `ignore` with `LowCardinality` argument. Fixes [#14275](https://github.com/ClickHouse/ClickHouse/issues/14275). [#19016](https://github.com/ClickHouse/ClickHouse/pull/19016) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19046](https://github.com/ClickHouse/ClickHouse/issues/19046): Make sure `groupUniqArray` returns correct type for argument of Enum type. This closes [#17875](https://github.com/ClickHouse/ClickHouse/issues/17875). [#19019](https://github.com/ClickHouse/ClickHouse/pull/19019) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19126](https://github.com/ClickHouse/ClickHouse/issues/19126): Restrict `MODIFY TTL` queries for `MergeTree` tables created in old syntax. Previously the query succeeded, but actually it had no effect. [#19064](https://github.com/ClickHouse/ClickHouse/pull/19064) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#19121](https://github.com/ClickHouse/ClickHouse/issues/19121): Simplify the implementation of `tupleHammingDistance`. Support for tuples of any equal length. Fixes [#19029](https://github.com/ClickHouse/ClickHouse/issues/19029). [#19084](https://github.com/ClickHouse/ClickHouse/pull/19084) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19150](https://github.com/ClickHouse/ClickHouse/issues/19150): Fix startup bug when clickhouse was not able to read compression codec from `LowCardinality(Nullable(...))` and throws exception `Attempt to read after EOF`. Fixes [#18340](https://github.com/ClickHouse/ClickHouse/issues/18340). [#19101](https://github.com/ClickHouse/ClickHouse/pull/19101) ([alesapin](https://github.com/alesapin)). +* Backported in [#19177](https://github.com/ClickHouse/ClickHouse/issues/19177): Fix bug in merge tree data writer which can lead to marks with bigger size than fixed granularity size. Fixes [#18913](https://github.com/ClickHouse/ClickHouse/issues/18913). [#19123](https://github.com/ClickHouse/ClickHouse/pull/19123) ([alesapin](https://github.com/alesapin)). +* Backported in [#19179](https://github.com/ClickHouse/ClickHouse/issues/19179): Fix infinite reading from file in `ORC` format (was introduced in [#10580](https://github.com/ClickHouse/ClickHouse/issues/10580)). Fixes [#19095](https://github.com/ClickHouse/ClickHouse/issues/19095). [#19134](https://github.com/ClickHouse/ClickHouse/pull/19134) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19178](https://github.com/ClickHouse/ClickHouse/issues/19178): - Split RemoteQueryExecutorReadContext into module part - Fix leaking of pipe fd for `async_socket_for_remote`. [#19153](https://github.com/ClickHouse/ClickHouse/pull/19153) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.1.3.32-stable.md b/docs/changelogs/v21.1.3.32-stable.md new file mode 100644 index 00000000000..ea4c9fd0fe6 --- /dev/null +++ b/docs/changelogs/v21.1.3.32-stable.md @@ -0,0 +1,31 @@ +### ClickHouse release v21.1.3.32-stable FIXME as compared to v21.1.2.15-stable + +#### Bug Fix +* Backported in [#19654](https://github.com/ClickHouse/ClickHouse/issues/19654): fix data type convert issue for mysql engine ... [#18124](https://github.com/ClickHouse/ClickHouse/pull/18124) ([bo zeng](https://github.com/mis98zb)). +* Backported in [#19423](https://github.com/ClickHouse/ClickHouse/issues/19423): Disable constant folding for subqueries on the analysis stage, when the result cannot be calculated. [#18446](https://github.com/ClickHouse/ClickHouse/pull/18446) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19458](https://github.com/ClickHouse/ClickHouse/issues/19458): Fixed `There is no checkpoint` error when inserting data through http interface using `Template` or `CustomSeparated` format. Fixes [#19021](https://github.com/ClickHouse/ClickHouse/issues/19021). [#19072](https://github.com/ClickHouse/ClickHouse/pull/19072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19271](https://github.com/ClickHouse/ClickHouse/issues/19271): Fix bug when concurrent `ALTER` and `DROP` queries may hang while processing ReplicatedMergeTree table. [#19237](https://github.com/ClickHouse/ClickHouse/pull/19237) ([alesapin](https://github.com/alesapin)). +* Backported in [#19425](https://github.com/ClickHouse/ClickHouse/issues/19425): Fix error `Cannot convert column now64() because it is constant but values of constants are different in source and result`. Continuation of [#7156](https://github.com/ClickHouse/ClickHouse/issues/7156). [#19316](https://github.com/ClickHouse/ClickHouse/pull/19316) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19456](https://github.com/ClickHouse/ClickHouse/issues/19456): Fixed possible wrong result or segfault on aggregation when Materialized View and its target table have different structure. Fixes [#18063](https://github.com/ClickHouse/ClickHouse/issues/18063). [#19322](https://github.com/ClickHouse/ClickHouse/pull/19322) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19554](https://github.com/ClickHouse/ClickHouse/issues/19554): Fix system.parts _state column (LOGICAL_ERROR when querying this column, due to incorrect order). [#19346](https://github.com/ClickHouse/ClickHouse/pull/19346) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19506](https://github.com/ClickHouse/ClickHouse/issues/19506): Added `cast`, `accurateCast`, `accurateCastOrNull` performance tests. [#19354](https://github.com/ClickHouse/ClickHouse/pull/19354) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#19471](https://github.com/ClickHouse/ClickHouse/issues/19471): - Fix default value in join types with non-zero default (e.g. some Enums). Closes [#18197](https://github.com/ClickHouse/ClickHouse/issues/18197). [#19360](https://github.com/ClickHouse/ClickHouse/pull/19360) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#19438](https://github.com/ClickHouse/ClickHouse/issues/19438): Fix possible buffer overflow in Uber H3 library. See https://github.com/uber/h3/issues/392. This closes [#19219](https://github.com/ClickHouse/ClickHouse/issues/19219). [#19383](https://github.com/ClickHouse/ClickHouse/pull/19383) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19485](https://github.com/ClickHouse/ClickHouse/issues/19485): Uninitialized memory read was possible in encrypt/decrypt functions if empty string was passed as IV. This closes [#19391](https://github.com/ClickHouse/ClickHouse/issues/19391). [#19397](https://github.com/ClickHouse/ClickHouse/pull/19397) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19615](https://github.com/ClickHouse/ClickHouse/issues/19615): Fixed very rare bug that might cause mutation to hang after `DROP/DETACH/REPLACE/MOVE PARTITION`. It was partially fixed by [#15537](https://github.com/ClickHouse/ClickHouse/issues/15537) for the most cases. [#19443](https://github.com/ClickHouse/ClickHouse/pull/19443) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19507](https://github.com/ClickHouse/ClickHouse/issues/19507): Buffer overflow (on memory read) was possible if `addMonth` function was called with specifically crafted arguments. This fixes [#19441](https://github.com/ClickHouse/ClickHouse/issues/19441). This fixes [#19413](https://github.com/ClickHouse/ClickHouse/issues/19413). [#19472](https://github.com/ClickHouse/ClickHouse/pull/19472) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19539](https://github.com/ClickHouse/ClickHouse/issues/19539): Fix SIGSEGV with merge_tree_min_rows_for_concurrent_read/merge_tree_min_bytes_for_concurrent_read=0/UINT64_MAX. [#19528](https://github.com/ClickHouse/ClickHouse/pull/19528) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19640](https://github.com/ClickHouse/ClickHouse/issues/19640): Query CREATE DICTIONARY id expression fix. [#19571](https://github.com/ClickHouse/ClickHouse/pull/19571) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#19636](https://github.com/ClickHouse/ClickHouse/issues/19636): `DROP/DETACH TABLE table ON CLUSTER cluster SYNC` query might hang, it's fixed. Fixes [#19568](https://github.com/ClickHouse/ClickHouse/issues/19568). [#19572](https://github.com/ClickHouse/ClickHouse/pull/19572) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19650](https://github.com/ClickHouse/ClickHouse/issues/19650): Fix use-after-free of the CompressedWriteBuffer in Connection after disconnect. [#19599](https://github.com/ClickHouse/ClickHouse/pull/19599) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#19738](https://github.com/ClickHouse/ClickHouse/issues/19738): Fix wrong result of function `neighbor` for `LowCardinality` argument. Fixes [#10333](https://github.com/ClickHouse/ClickHouse/issues/10333). [#19617](https://github.com/ClickHouse/ClickHouse/pull/19617) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19700](https://github.com/ClickHouse/ClickHouse/issues/19700): Some functions with big integers may cause segfault. Big integers is experimental feature. This closes [#19667](https://github.com/ClickHouse/ClickHouse/issues/19667). [#19672](https://github.com/ClickHouse/ClickHouse/pull/19672) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19742](https://github.com/ClickHouse/ClickHouse/issues/19742): Fix a segmentation fault in `bitmapAndnot` function. Fixes [#19668](https://github.com/ClickHouse/ClickHouse/issues/19668). [#19713](https://github.com/ClickHouse/ClickHouse/pull/19713) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#19782](https://github.com/ClickHouse/ClickHouse/issues/19782): Fix crash when nested column name was used in `WHERE` or `PREWHERE`. Fixes [#19755](https://github.com/ClickHouse/ClickHouse/issues/19755). [#19763](https://github.com/ClickHouse/ClickHouse/pull/19763) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#19870](https://github.com/ClickHouse/ClickHouse/issues/19870): Fixed stack overflow when using accurate comparison of arithmetic type with string type. [#19773](https://github.com/ClickHouse/ClickHouse/pull/19773) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#19814](https://github.com/ClickHouse/ClickHouse/issues/19814): In previous versions, unusual arguments for function arrayEnumerateUniq may cause crash or infinite loop. This closes [#19787](https://github.com/ClickHouse/ClickHouse/issues/19787). [#19788](https://github.com/ClickHouse/ClickHouse/pull/19788) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19835](https://github.com/ClickHouse/ClickHouse/issues/19835): Fix filtering by UInt8 greater than 127. [#19799](https://github.com/ClickHouse/ClickHouse/pull/19799) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#19910](https://github.com/ClickHouse/ClickHouse/issues/19910): Fix crash when pushing down predicates to union distinct subquery. This fixes [#19855](https://github.com/ClickHouse/ClickHouse/issues/19855). [#19861](https://github.com/ClickHouse/ClickHouse/pull/19861) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#19938](https://github.com/ClickHouse/ClickHouse/issues/19938): Deadlock was possible if system.text_log is enabled. This fixes [#19874](https://github.com/ClickHouse/ClickHouse/issues/19874). [#19875](https://github.com/ClickHouse/ClickHouse/pull/19875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19934](https://github.com/ClickHouse/ClickHouse/issues/19934): BloomFilter index crash fix. Fixes [#19757](https://github.com/ClickHouse/ClickHouse/issues/19757). [#19884](https://github.com/ClickHouse/ClickHouse/pull/19884) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.1.4.46-stable.md b/docs/changelogs/v21.1.4.46-stable.md new file mode 100644 index 00000000000..3033c5edd21 --- /dev/null +++ b/docs/changelogs/v21.1.4.46-stable.md @@ -0,0 +1,22 @@ +### ClickHouse release v21.1.4.46-stable FIXME as compared to v21.1.3.32-stable + +#### Bug Fix +* Backported in [#19983](https://github.com/ClickHouse/ClickHouse/issues/19983): Background thread which executes `ON CLUSTER` queries might hang waiting for dropped replicated table to do something. It's fixed. [#19684](https://github.com/ClickHouse/ClickHouse/pull/19684) ([yiguolei](https://github.com/yiguolei)). +* Backported in [#20238](https://github.com/ClickHouse/ClickHouse/issues/20238): Fix a bug that moving pieces to destination table may failed in case of launching multiple clickhouse-copiers. [#19743](https://github.com/ClickHouse/ClickHouse/pull/19743) ([madianjun](https://github.com/mdianjun)). +* Backported in [#20074](https://github.com/ClickHouse/ClickHouse/issues/20074): Fix starting the server with tables having default expressions containing dictGet(). Allow getting return type of dictGet() without loading dictionary. [#19805](https://github.com/ClickHouse/ClickHouse/pull/19805) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#19997](https://github.com/ClickHouse/ClickHouse/issues/19997): - Fix a segfault in function `fromModifiedJulianDay` when the argument type is `Nullable(T)` for any integral types other than Int32. [#19959](https://github.com/ClickHouse/ClickHouse/pull/19959) ([PHO](https://github.com/depressed-pho)). +* Backported in [#20122](https://github.com/ClickHouse/ClickHouse/issues/20122): MaterializeMySQL: Fix replication for statements that update several tables. [#20066](https://github.com/ClickHouse/ClickHouse/pull/20066) ([Håvard Kvålen](https://github.com/havardk)). +* Backported in [#20297](https://github.com/ClickHouse/ClickHouse/issues/20297): * Bugfix in StorageJoin. [#20079](https://github.com/ClickHouse/ClickHouse/pull/20079) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#20390](https://github.com/ClickHouse/ClickHouse/issues/20390): The `MongoDB` table engine now establishes connection only when it's going to read data. `ATTACH TABLE` won't try to connect anymore. [#20110](https://github.com/ClickHouse/ClickHouse/pull/20110) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#20148](https://github.com/ClickHouse/ClickHouse/issues/20148): Fix server crash after query with `if` function with `Tuple` type of then/else branches result. `Tuple` type must contain `Array` or another complex type. Fixes [#18356](https://github.com/ClickHouse/ClickHouse/issues/18356). [#20133](https://github.com/ClickHouse/ClickHouse/pull/20133) ([alesapin](https://github.com/alesapin)). +* Backported in [#20327](https://github.com/ClickHouse/ClickHouse/issues/20327): Fix rare server crash on config reload during the shutdown. Fixes [#19689](https://github.com/ClickHouse/ClickHouse/issues/19689). [#20224](https://github.com/ClickHouse/ClickHouse/pull/20224) ([alesapin](https://github.com/alesapin)). +* Backported in [#20331](https://github.com/ClickHouse/ClickHouse/issues/20331): Restrict to `DROP` or `RENAME` version column of `*CollapsingMergeTree` and `ReplacingMergeTree` table engines. [#20300](https://github.com/ClickHouse/ClickHouse/pull/20300) ([alesapin](https://github.com/alesapin)). +* Backported in [#20363](https://github.com/ClickHouse/ClickHouse/issues/20363): Fix too often retries of failed background tasks for `ReplicatedMergeTree` table engines family. This could lead to too verbose logging and increased CPU load. Fixes [#20203](https://github.com/ClickHouse/ClickHouse/issues/20203). [#20335](https://github.com/ClickHouse/ClickHouse/pull/20335) ([alesapin](https://github.com/alesapin)). +* Backported in [#20378](https://github.com/ClickHouse/ClickHouse/issues/20378): Fix incorrect result of binary operations between two constant decimals of different scale. Fixes [#20283](https://github.com/ClickHouse/ClickHouse/issues/20283). [#20339](https://github.com/ClickHouse/ClickHouse/pull/20339) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#20375](https://github.com/ClickHouse/ClickHouse/issues/20375): Fix null dereference with `join_use_nulls=1`. [#20344](https://github.com/ClickHouse/ClickHouse/pull/20344) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#20359](https://github.com/ClickHouse/ClickHouse/issues/20359): Avoid invalid dereference in RANGE_HASHED() dictionary. [#20345](https://github.com/ClickHouse/ClickHouse/pull/20345) ([Azat Khuzhin](https://github.com/azat)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Backport [#20224](https://github.com/ClickHouse/ClickHouse/issues/20224) to 21.1: Fix access control manager destruction order"'. [#20395](https://github.com/ClickHouse/ClickHouse/pull/20395) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.1.5.4-stable.md b/docs/changelogs/v21.1.5.4-stable.md new file mode 100644 index 00000000000..c67724a4512 --- /dev/null +++ b/docs/changelogs/v21.1.5.4-stable.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.1.5.4-stable FIXME as compared to v21.1.4.46-stable + +#### Bug Fix +* Backported in [#20678](https://github.com/ClickHouse/ClickHouse/issues/20678): Mark distributed batch as broken in case of empty data block in one of files. [#19449](https://github.com/ClickHouse/ClickHouse/pull/19449) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#20646](https://github.com/ClickHouse/ClickHouse/issues/20646): The function `greatCircleAngle` returned inaccurate results in previous versions. This closes [#19769](https://github.com/ClickHouse/ClickHouse/issues/19769). [#19789](https://github.com/ClickHouse/ClickHouse/pull/19789) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#20507](https://github.com/ClickHouse/ClickHouse/issues/20507): Fixed the behavior when in case of broken JSON we tried to read the whole file into memory which leads to exception from the allocator. Fixes [#19719](https://github.com/ClickHouse/ClickHouse/issues/19719). [#20286](https://github.com/ClickHouse/ClickHouse/pull/20286) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#20618](https://github.com/ClickHouse/ClickHouse/issues/20618): Check if table function `view` is used in expression list and throw an error. This fixes [#20342](https://github.com/ClickHouse/ClickHouse/issues/20342). [#20350](https://github.com/ClickHouse/ClickHouse/pull/20350) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#20488](https://github.com/ClickHouse/ClickHouse/issues/20488): Fix `LOGICAL_ERROR` for `join_use_nulls=1` when JOIN contains const from SELECT. [#20461](https://github.com/ClickHouse/ClickHouse/pull/20461) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#20886](https://github.com/ClickHouse/ClickHouse/issues/20886): Fix subquery with union distinct and limit clause. close [#20597](https://github.com/ClickHouse/ClickHouse/issues/20597). [#20610](https://github.com/ClickHouse/ClickHouse/pull/20610) ([flynn](https://github.com/ucasfl)). +* Backported in [#20992](https://github.com/ClickHouse/ClickHouse/issues/20992): Fix usage of `-Distinct` combinator with `-State` combinator in aggregate functions. [#20866](https://github.com/ClickHouse/ClickHouse/pull/20866) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#20988](https://github.com/ClickHouse/ClickHouse/issues/20988): `USE database;` query did not work when using MySQL 5.7 client to connect to ClickHouse server, it's fixed. Fixes [#18926](https://github.com/ClickHouse/ClickHouse/issues/18926). [#20878](https://github.com/ClickHouse/ClickHouse/pull/20878) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.1.6.13-stable.md b/docs/changelogs/v21.1.6.13-stable.md new file mode 100644 index 00000000000..547cd38a06f --- /dev/null +++ b/docs/changelogs/v21.1.6.13-stable.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.1.6.13-stable FIXME as compared to v21.1.5.4-stable + +#### Bug Fix +* Backported in [#20637](https://github.com/ClickHouse/ClickHouse/issues/20637): Fix rare bug when some replicated operations (like mutation) cannot process some parts after data corruption. Fixes [#19593](https://github.com/ClickHouse/ClickHouse/issues/19593). [#19702](https://github.com/ClickHouse/ClickHouse/pull/19702) ([alesapin](https://github.com/alesapin)). +* Backported in [#20573](https://github.com/ClickHouse/ClickHouse/issues/20573): Fix crash which could happen if unknown packet was received from remove query (was introduced in [#17868](https://github.com/ClickHouse/ClickHouse/issues/17868)). [#20547](https://github.com/ClickHouse/ClickHouse/pull/20547) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21005](https://github.com/ClickHouse/ClickHouse/issues/21005): Fix 'Empty task was returned from async task queue' on query cancellation. [#20881](https://github.com/ClickHouse/ClickHouse/pull/20881) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21131](https://github.com/ClickHouse/ClickHouse/issues/21131): Fixed behaviour, when `ALTER MODIFY COLUMN` created mutation, that will knowingly fail. [#21007](https://github.com/ClickHouse/ClickHouse/pull/21007) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#21250](https://github.com/ClickHouse/ClickHouse/issues/21250): - Block parallel insertions into storage join. [#21009](https://github.com/ClickHouse/ClickHouse/pull/21009) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#21068](https://github.com/ClickHouse/ClickHouse/issues/21068): Out of bound memory access was possible when formatting specifically crafted out of range value of type `DateTime64`. This closes [#20494](https://github.com/ClickHouse/ClickHouse/issues/20494). This closes [#20543](https://github.com/ClickHouse/ClickHouse/issues/20543). [#21023](https://github.com/ClickHouse/ClickHouse/pull/21023) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#21229](https://github.com/ClickHouse/ClickHouse/issues/21229): Fixes [#21112](https://github.com/ClickHouse/ClickHouse/issues/21112). Fixed bug that could cause duplicates with insert query (if one of the callbacks came a little too late). [#21138](https://github.com/ClickHouse/ClickHouse/pull/21138) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#21277](https://github.com/ClickHouse/ClickHouse/issues/21277): Fix bug with `join_use_nulls` and joining `TOTALS` from subqueries. This closes [#19362](https://github.com/ClickHouse/ClickHouse/issues/19362) and [#21137](https://github.com/ClickHouse/ClickHouse/issues/21137). [#21248](https://github.com/ClickHouse/ClickHouse/pull/21248) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.1.7.1-stable.md b/docs/changelogs/v21.1.7.1-stable.md new file mode 100644 index 00000000000..371efb8d5df --- /dev/null +++ b/docs/changelogs/v21.1.7.1-stable.md @@ -0,0 +1,11 @@ +### ClickHouse release v21.1.7.1-stable FIXME as compared to v21.1.6.13-stable + +#### Bug Fix +* Backported in [#21261](https://github.com/ClickHouse/ClickHouse/issues/21261): fix default_replica_path and default_replica_name values are useless on Replicated(*)MergeTree engine when the engine needs specify other parameters. [#21060](https://github.com/ClickHouse/ClickHouse/pull/21060) ([mxzlxy](https://github.com/mxzlxy)). +* Backported in [#21155](https://github.com/ClickHouse/ClickHouse/issues/21155): fix bug related to cast tuple to map. Closes [#21029](https://github.com/ClickHouse/ClickHouse/issues/21029). [#21120](https://github.com/ClickHouse/ClickHouse/pull/21120) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#21233](https://github.com/ClickHouse/ClickHouse/issues/21233): Now mutations allowed only for table engines that support them (MergeTree family, Memory, MaterializedView). Other engines will report a more clear error. Fixes [#21168](https://github.com/ClickHouse/ClickHouse/issues/21168). [#21183](https://github.com/ClickHouse/ClickHouse/pull/21183) ([alesapin](https://github.com/alesapin)). +* Backported in [#21428](https://github.com/ClickHouse/ClickHouse/issues/21428): Fix crash in `EXPLAIN` for query with `UNION`. Fixes [#20876](https://github.com/ClickHouse/ClickHouse/issues/20876), [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170). [#21246](https://github.com/ClickHouse/ClickHouse/pull/21246) ([flynn](https://github.com/ucasfl)). +* Backported in [#21408](https://github.com/ClickHouse/ClickHouse/issues/21408): Fix redundant reconnects to ZooKeeper and the possibility of two active sessions for a single clickhouse server. Both problems introduced in [#14678](https://github.com/ClickHouse/ClickHouse/issues/14678). [#21264](https://github.com/ClickHouse/ClickHouse/pull/21264) ([alesapin](https://github.com/alesapin)). +* Backported in [#21549](https://github.com/ClickHouse/ClickHouse/issues/21549): Now `ALTER MODIFY COLUMN` queries will correctly affect changes in partition key, skip indices, TTLs, and so on. Fixes [#13675](https://github.com/ClickHouse/ClickHouse/issues/13675). [#21334](https://github.com/ClickHouse/ClickHouse/pull/21334) ([alesapin](https://github.com/alesapin)). +* Backported in [#21377](https://github.com/ClickHouse/ClickHouse/issues/21377): Fix error `Bad cast from type ... to DB::ColumnLowCardinality` while inserting into table with `LowCardinality` column from `Values` format. Fixes [#21140](https://github.com/ClickHouse/ClickHouse/issues/21140). [#21357](https://github.com/ClickHouse/ClickHouse/pull/21357) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.1.8.30-stable.md b/docs/changelogs/v21.1.8.30-stable.md new file mode 100644 index 00000000000..0859cc8ccbd --- /dev/null +++ b/docs/changelogs/v21.1.8.30-stable.md @@ -0,0 +1,29 @@ +### ClickHouse release v21.1.8.30-stable FIXME as compared to v21.1.7.1-stable + +#### Bug Fix +* Backported in [#21205](https://github.com/ClickHouse/ClickHouse/issues/21205): Fix the metadata leak when the Replicated*MergeTree with custom (non default) ZooKeeper cluster is dropped. [#21119](https://github.com/ClickHouse/ClickHouse/pull/21119) ([fastio](https://github.com/fastio)). +* Backported in [#21161](https://github.com/ClickHouse/ClickHouse/issues/21161): Fix `input_format_null_as_default` take effective when types are nullable. This fixes [#21116](https://github.com/ClickHouse/ClickHouse/issues/21116) . [#21121](https://github.com/ClickHouse/ClickHouse/pull/21121) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#21926](https://github.com/ClickHouse/ClickHouse/issues/21926): Fix Avro format parsing for Kafka. Fixes [#21437](https://github.com/ClickHouse/ClickHouse/issues/21437). [#21438](https://github.com/ClickHouse/ClickHouse/pull/21438) ([Ilya Golshtein](https://github.com/ilejn)). +* Backported in [#22189](https://github.com/ClickHouse/ClickHouse/issues/22189): Fixed race on SSL object inside SecureSocket in Poco. [#21456](https://github.com/ClickHouse/ClickHouse/pull/21456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#22317](https://github.com/ClickHouse/ClickHouse/issues/22317): Remove unknown columns from joined table in where for queries to external database engines (MySQL, PostgreSQL). close [#14614](https://github.com/ClickHouse/ClickHouse/issues/14614), close [#19288](https://github.com/ClickHouse/ClickHouse/issues/19288) (dup), close [#19645](https://github.com/ClickHouse/ClickHouse/issues/19645) (dup). [#21640](https://github.com/ClickHouse/ClickHouse/pull/21640) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#21797](https://github.com/ClickHouse/ClickHouse/issues/21797): Fix distributed requests cancellation (for example simple select from multiple shards with limit, i.e. `select * from remote('127.{2,3}', system.numbers) limit 100`) with `async_socket_for_remote=1`. [#21643](https://github.com/ClickHouse/ClickHouse/pull/21643) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22287](https://github.com/ClickHouse/ClickHouse/issues/22287): Start accepting connections after DDLWorker and dictionaries initialization. [#21676](https://github.com/ClickHouse/ClickHouse/pull/21676) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21810](https://github.com/ClickHouse/ClickHouse/issues/21810): Fix bug for ReplicatedMerge table engines when `ALTER MODIFY COLUMN` query doesn't change the type of decimal column if its size (32 bit or 64 bit) doesn't change. [#21728](https://github.com/ClickHouse/ClickHouse/pull/21728) ([alesapin](https://github.com/alesapin)). +* Backported in [#21880](https://github.com/ClickHouse/ClickHouse/issues/21880): Fix possible crashes in aggregate functions with combinator Distinct, while using two-level aggregation. This is a follow-up fix of https://github.com/ClickHouse/ClickHouse/pull/18365 . Can only reproduced in production env. No test case available yet. cc @CurtizJ. [#21818](https://github.com/ClickHouse/ClickHouse/pull/21818) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#22052](https://github.com/ClickHouse/ClickHouse/issues/22052): Fix deadlock in first catboost model execution. Closes [#13832](https://github.com/ClickHouse/ClickHouse/issues/13832). [#21844](https://github.com/ClickHouse/ClickHouse/pull/21844) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#21981](https://github.com/ClickHouse/ClickHouse/issues/21981): Reverted [#15454](https://github.com/ClickHouse/ClickHouse/issues/15454) that may cause significant increase in memory usage while loading external dictionaries of hashed type. This closes [#21935](https://github.com/ClickHouse/ClickHouse/issues/21935). [#21948](https://github.com/ClickHouse/ClickHouse/pull/21948) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#22465](https://github.com/ClickHouse/ClickHouse/issues/22465): In rare case, merge for `CollapsingMergeTree` may create granule with `index_granularity + 1` rows. Because of this, internal check, added in [#18928](https://github.com/ClickHouse/ClickHouse/issues/18928) (affects 21.2 and 21.3), may fail with error `Incomplete granules are not allowed while blocks are granules size`. This error did not allow parts to merge. [#21976](https://github.com/ClickHouse/ClickHouse/pull/21976) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#22149](https://github.com/ClickHouse/ClickHouse/issues/22149): The function `decrypt` was lacking a check for the minimal size of data encrypted in AEAD mode. This closes [#21897](https://github.com/ClickHouse/ClickHouse/issues/21897). [#22064](https://github.com/ClickHouse/ClickHouse/pull/22064) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22283](https://github.com/ClickHouse/ClickHouse/issues/22283): Docker entrypoint: avoid chown of `.` in case when `LOG_PATH` is empty. Closes [#22100](https://github.com/ClickHouse/ClickHouse/issues/22100). [#22102](https://github.com/ClickHouse/ClickHouse/pull/22102) ([filimonov](https://github.com/filimonov)). +* Backported in [#22280](https://github.com/ClickHouse/ClickHouse/issues/22280): Fix waiting for `OPTIMIZE` and `ALTER` queries for `ReplicatedMergeTree` table engines. Now the query will not hang when the table was detached or restarted. [#22118](https://github.com/ClickHouse/ClickHouse/pull/22118) ([alesapin](https://github.com/alesapin)). +* Backported in [#22501](https://github.com/ClickHouse/ClickHouse/issues/22501): Fix query cancellation with `use_hedged_requests=0` and `async_socket_for_remote=1`. [#22183](https://github.com/ClickHouse/ClickHouse/pull/22183) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22368](https://github.com/ClickHouse/ClickHouse/issues/22368): Now clickhouse will not throw `LOGICAL_ERROR` exception when we try to mutate the already covered part. Fixes [#22013](https://github.com/ClickHouse/ClickHouse/issues/22013). [#22291](https://github.com/ClickHouse/ClickHouse/pull/22291) ([alesapin](https://github.com/alesapin)). +* Backported in [#22533](https://github.com/ClickHouse/ClickHouse/issues/22533): Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22462](https://github.com/ClickHouse/ClickHouse/issues/22462): Add (missing) memory accounting in parallel parsing routines. In previous versions OOM was possible when the resultset contains very large blocks of data. This closes [#22008](https://github.com/ClickHouse/ClickHouse/issues/22008). [#22425](https://github.com/ClickHouse/ClickHouse/pull/22425) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22555](https://github.com/ClickHouse/ClickHouse/issues/22555): Fix bug in partial merge join with `LowCardinality`. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#22608](https://github.com/ClickHouse/ClickHouse/issues/22608): Fix deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22578](https://github.com/ClickHouse/ClickHouse/issues/22578): Fix UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22650](https://github.com/ClickHouse/ClickHouse/issues/22650): Avoid UB in *Log engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22700](https://github.com/ClickHouse/ClickHouse/issues/22700): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). +* Backported in [#22739](https://github.com/ClickHouse/ClickHouse/issues/22739): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.1.9.41-stable.md b/docs/changelogs/v21.1.9.41-stable.md new file mode 100644 index 00000000000..2a94073b810 --- /dev/null +++ b/docs/changelogs/v21.1.9.41-stable.md @@ -0,0 +1,17 @@ +### ClickHouse release v21.1.9.41-stable FIXME as compared to v21.1.8.30-stable + +#### Improvement +* Backported in [#22817](https://github.com/ClickHouse/ClickHouse/issues/22817): Make FQDN and other DNS related functions work correctly in alpine images. [#20336](https://github.com/ClickHouse/ClickHouse/pull/20336) ([filimonov](https://github.com/filimonov)). +* Backported in [#22810](https://github.com/ClickHouse/ClickHouse/issues/22810): If PODArray was instantiated with element size that is neither a fraction or a multiple of 16, buffer overflow was possible. No bugs in current releases exist. [#21533](https://github.com/ClickHouse/ClickHouse/pull/21533) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#22967](https://github.com/ClickHouse/ClickHouse/issues/22967): Fix very rare bug when quorum insert with `quorum_parallel=1` is not really "quorum" because of deduplication. [#18215](https://github.com/ClickHouse/ClickHouse/pull/18215) ([filimonov](https://github.com/filimonov)). +* Backported in [#22088](https://github.com/ClickHouse/ClickHouse/issues/22088): In case if query has constant `WHERE` condition, and setting `optimize_skip_unused_shards` enabled, all shards may be skipped and query could return incorrect empty result. [#21550](https://github.com/ClickHouse/ClickHouse/pull/21550) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#21857](https://github.com/ClickHouse/ClickHouse/issues/21857): Fix possible error ` Cannot find column` when `optimize_skip_unused_shards` is enabled and zero shards are used. [#21579](https://github.com/ClickHouse/ClickHouse/pull/21579) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22758](https://github.com/ClickHouse/ClickHouse/issues/22758): Fix usage of function `map` in distributed queries. [#22588](https://github.com/ClickHouse/ClickHouse/pull/22588) ([foolchi](https://github.com/foolchi)). +* Backported in [#22890](https://github.com/ClickHouse/ClickHouse/issues/22890): Fix approx total rows accounting for reverse reading from MergeTree. [#22726](https://github.com/ClickHouse/ClickHouse/pull/22726) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22919](https://github.com/ClickHouse/ClickHouse/issues/22919): Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### Build/Testing/Packaging Improvement +* Backported in [#22813](https://github.com/ClickHouse/ClickHouse/issues/22813): Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.10.1.8013-prestable.md b/docs/changelogs/v21.10.1.8013-prestable.md new file mode 100644 index 00000000000..d3e06c056cf --- /dev/null +++ b/docs/changelogs/v21.10.1.8013-prestable.md @@ -0,0 +1,138 @@ +### ClickHouse release v21.10.1.8013-prestable FIXME as compared to v21.9.1.7770-prestable + +#### Backward Incompatible Change +* Fix the issue that in case of some sophisticated query with column aliases identical to the names of expressions, bad cast may happen. This fixes [#25447](https://github.com/ClickHouse/ClickHouse/issues/25447). This fixes [#26914](https://github.com/ClickHouse/ClickHouse/issues/26914). This fix may introduce backward incompatibility: if there are different expressions with identical names, exception will be thrown. It may break some rare cases when `enable_optimize_predicate_expression` is set. [#26639](https://github.com/ClickHouse/ClickHouse/pull/26639) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not output trailing zeros in text representation of `Decimal` types. Example: `1.23` will be printed instead of `1.230000` for decimal with scale 6. This closes [#15794](https://github.com/ClickHouse/ClickHouse/issues/15794). It may introduce slight incompatibility if your applications somehow relied on the trailing zeros. Serialization in output formats can be controlled with the setting `output_format_decimal_trailing_zeros`. Implementation of `toString` and casting to String is changed unconditionally. [#27680](https://github.com/ClickHouse/ClickHouse/pull/27680) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now MergeTreeSettings `replicated_max_parallel_sends`, `replicated_max_parallel_sends_for_table`, `replicated_max_parallel_fetches`, `replicated_max_parallel_fetches_for_table` do nothing. They never worked well and were replaced with `max_replicated_fetches_network_bandwidth`, `max_replicated_sends_network_bandwidth` and `background_fetches_pool_size`. [#28404](https://github.com/ClickHouse/ClickHouse/pull/28404) ([alesapin](https://github.com/alesapin)). + +#### New Feature +* Generate a unique server uuid when server starts. [#20089](https://github.com/ClickHouse/ClickHouse/pull/20089) ([Bharat Nallan](https://github.com/bharatnc)). +* Added new commands BACKUP and RESTORE. [#21945](https://github.com/ClickHouse/ClickHouse/pull/21945) ([Vitaly Baranov](https://github.com/vitlibar)). +* Partitioned write into s3 table function. [#23051](https://github.com/ClickHouse/ClickHouse/pull/23051) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Implementation of short circuit function evaluation, closes [#12587](https://github.com/ClickHouse/ClickHouse/issues/12587). Add settings `short_circuit_function_evaluation` to configure short circuit function evaluation. [#23367](https://github.com/ClickHouse/ClickHouse/pull/23367) ([Kruglov Pavel](https://github.com/Avogar)). +* Add feature for creating user-defined functions. [#23978](https://github.com/ClickHouse/ClickHouse/pull/23978) ([Realist007](https://github.com/Realist007)). +* Add support for INTERSECT, EXCEPT, ANY, ALL operators. [#24757](https://github.com/ClickHouse/ClickHouse/pull/24757) ([Kirill Ershov](https://github.com/zdikov)). +* IDisk interface to store data on web server of static files. Closes [#23982](https://github.com/ClickHouse/ClickHouse/issues/23982). [#25251](https://github.com/ClickHouse/ClickHouse/pull/25251) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Introduce lz4 compression for import / export. [#25310](https://github.com/ClickHouse/ClickHouse/pull/25310) ([Bharat Nallan](https://github.com/bharatnc)). +* Support the case when the data is enclosed in array in JSONAsString input format. Closes [#25517](https://github.com/ClickHouse/ClickHouse/issues/25517). [#25633](https://github.com/ClickHouse/ClickHouse/pull/25633) ([Kruglov Pavel](https://github.com/Avogar)). +* Add new column `last_queue_update_exception` to `system.replicas` table. [#26843](https://github.com/ClickHouse/ClickHouse/pull/26843) ([nvartolomei](https://github.com/nvartolomei)). +* ALTER TABLE ... MATERIALIZE COLUMN. [#27038](https://github.com/ClickHouse/ClickHouse/pull/27038) ([Vladimir Chebotarev](https://github.com/excitoon)). +* - Add replicated storage of user, roles, row policies, quotas and settings profiles through ZooKeeper (experimental). [#27426](https://github.com/ClickHouse/ClickHouse/pull/27426) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Allow positional arguments under setting `enable_positional_arguments`. Closes [#2592](https://github.com/ClickHouse/ClickHouse/issues/2592). [#27530](https://github.com/ClickHouse/ClickHouse/pull/27530) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Added ComplexKeyRangeHashed dictionary. Closes [#22029](https://github.com/ClickHouse/ClickHouse/issues/22029). [#27629](https://github.com/ClickHouse/ClickHouse/pull/27629) ([Maksim Kita](https://github.com/kitaisreal)). +* add conversion functions between snowflake id and dateTime(dateTime64) Close [#27058](https://github.com/ClickHouse/ClickHouse/issues/27058). [#27704](https://github.com/ClickHouse/ClickHouse/pull/27704) ([jasine](https://github.com/jasine)). +* Add feature for creating user-defined functions as lambda expressions. Syntax `CREATE FUNCTION {function_name} as ({parameters}) -> {function core}`. Example `CREATE FUNCTION plus_one as (a) -> a + 1`. Authors @Realist007. [#27796](https://github.com/ClickHouse/ClickHouse/pull/27796) ([Maksim Kita](https://github.com/kitaisreal)). +* Add `getServerPort` function to allow getting server port. When the port is not used by the server, throw an exception. [#27900](https://github.com/ClickHouse/ClickHouse/pull/27900) ([Amos Bird](https://github.com/amosbird)). +* Accept user settings related to file formats in `SETTINGS` clause in `CREATE` query. This closes [#27580](https://github.com/ClickHouse/ClickHouse/issues/27580). [#28037](https://github.com/ClickHouse/ClickHouse/pull/28037) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add a system table of table_views, convenient to query the dependency relationship between tables and views. [#28082](https://github.com/ClickHouse/ClickHouse/pull/28082) ([zhongyuankai](https://github.com/zhongyuankai)). +* Added `executable` storage engine and table function. Authors @ruct. [#28102](https://github.com/ClickHouse/ClickHouse/pull/28102) ([Maksim Kita](https://github.com/kitaisreal)). +* Added `ExecutablePool` storage. [#28518](https://github.com/ClickHouse/ClickHouse/pull/28518) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Performance Improvement +* Introducing two checks in `sequenceMatch` and `sequenceCount` that allow for early exit when some deterministic part of the sequence pattern is missing from the events list. This change unlocks many queries that would previously fail due to reaching operations cap, and generally speeds up the pipeline. [#27729](https://github.com/ClickHouse/ClickHouse/pull/27729) ([Jakub Kuklis](https://github.com/jkuklis)). +* Make `hasAll` filter condition leverage bloom filter data-skipping indexes. [#27984](https://github.com/ClickHouse/ClickHouse/pull/27984) ([Braulio Valdivielso Martínez](https://github.com/BraulioVM)). +* Speed up sumIf and countIf aggregation functions. [#28272](https://github.com/ClickHouse/ClickHouse/pull/28272) ([Raúl Marín](https://github.com/Algunenano)). +* Enhance primary key analysis with always monotonic information of binary functions, notably non-zero constant division. [#28302](https://github.com/ClickHouse/ClickHouse/pull/28302) ([Amos Bird](https://github.com/amosbird)). + +#### Improvement +* Create virtual projection for `min_max` indices. Now, when `allow_experimental_projection_optimization ` is enabled, queries will use minmax index instead of reading a part when possible. [#26286](https://github.com/ClickHouse/ClickHouse/pull/26286) ([Amos Bird](https://github.com/amosbird)). +* improve Materialize TTL by recalculating ttl.txt only without actual ttl action. [#27019](https://github.com/ClickHouse/ClickHouse/pull/27019) ([lthaooo](https://github.com/lthaooo)). +* Improved the existence condition judgment and empty string node judgment when clickhouse-keeper creates znode. [#27125](https://github.com/ClickHouse/ClickHouse/pull/27125) ([小路](https://github.com/nicelulu)). +* Don't silently ignore errors and don't count delays in `ReadBufferFromS3`. [#27484](https://github.com/ClickHouse/ClickHouse/pull/27484) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Add `log_queries_probability` setting that allows user to write to query_log only a sample of queries. Closes [#16609](https://github.com/ClickHouse/ClickHouse/issues/16609). [#27527](https://github.com/ClickHouse/ClickHouse/pull/27527) ([Nikolay Degterinsky](https://github.com/evillique)). +* Disable arrayJoin on partition expressions. [#27648](https://github.com/ClickHouse/ClickHouse/pull/27648) ([Raúl Marín](https://github.com/Algunenano)). +* - Add `FROM INFILE` command. [#27655](https://github.com/ClickHouse/ClickHouse/pull/27655) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Enables query parameters to be passed in the body of http requests. [#27706](https://github.com/ClickHouse/ClickHouse/pull/27706) ([Hermano Lustosa](https://github.com/hllustosa)). +* Remove duplicate index analysis and avoid possible invalid limit checks during projection analysis. [#27742](https://github.com/ClickHouse/ClickHouse/pull/27742) ([Amos Bird](https://github.com/amosbird)). +* Add aggregate function `quantileBFloat16Weighted` similarly to other quantile...Weighted functions. This closes [#27745](https://github.com/ClickHouse/ClickHouse/issues/27745). [#27758](https://github.com/ClickHouse/ClickHouse/pull/27758) ([Ivan Novitskiy](https://github.com/RedClusive)). +* Now `ALTER MODIFY COLUM` DataType to `Nullable(DataType)` doesn't require mutation. [#27787](https://github.com/ClickHouse/ClickHouse/pull/27787) ([victorgao](https://github.com/kafka1991)). +* Allow symlinks for library dictionaty path. [#27815](https://github.com/ClickHouse/ClickHouse/pull/27815) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add function `SHA512`. [#27830](https://github.com/ClickHouse/ClickHouse/pull/27830) ([zhanglistar](https://github.com/zhanglistar)). +* Use Multipart copy upload for large S3 objects. [#27858](https://github.com/ClickHouse/ClickHouse/pull/27858) ([ianton-ru](https://github.com/ianton-ru)). +* Improve remote query cancelation (in case of remote server abnormaly terminated). [#27881](https://github.com/ClickHouse/ClickHouse/pull/27881) ([Azat Khuzhin](https://github.com/azat)). +* Enable tcp_keep_alive_timeout by default. [#27882](https://github.com/ClickHouse/ClickHouse/pull/27882) ([Azat Khuzhin](https://github.com/azat)). +* Fix incorrect assertion during writing to StorageKafka. [#27885](https://github.com/ClickHouse/ClickHouse/pull/27885) ([Azat Khuzhin](https://github.com/azat)). +* Support lambda argument for APPLY column transformer which allows applying functions with more than one argument. This is for [#27877](https://github.com/ClickHouse/ClickHouse/issues/27877). [#27901](https://github.com/ClickHouse/ClickHouse/pull/27901) ([Amos Bird](https://github.com/amosbird)). +* Add interactive documentation in `clickhouse-client` about how to reset the password. This is useful in scenario when user has installed ClickHouse, set up the password and instantly forget it. See [#27750](https://github.com/ClickHouse/ClickHouse/issues/27750). [#27903](https://github.com/ClickHouse/ClickHouse/pull/27903) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to create dictionaries with empty attributes list. [#27905](https://github.com/ClickHouse/ClickHouse/pull/27905) ([Maksim Kita](https://github.com/kitaisreal)). +* Added `replication_wait_for_inactive_replica_timeout` setting. It allows to specify how long to wait for inactive replicas to execute `ALTER`/`OPTIMZE`/`TRUNCATE` query (default is 120 seconds). If `replication_alter_partitions_sync` is 2 and some replicas are not active for more than `replication_wait_for_inactive_replica_timeout` seconds, then `UNFINISHED` will be thrown. [#27931](https://github.com/ClickHouse/ClickHouse/pull/27931) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a setting `empty_result_for_aggregation_by_constant_keys_on_empty_set` to control the behavior of grouping by constant keys on empty set. This is to bring back the old baviour of [#6842](https://github.com/ClickHouse/ClickHouse/issues/6842). [#27932](https://github.com/ClickHouse/ClickHouse/pull/27932) ([Amos Bird](https://github.com/amosbird)). +* Lower restrictions for Enum data type to allow attaching compatible data. Closes [#26672](https://github.com/ClickHouse/ClickHouse/issues/26672). [#28028](https://github.com/ClickHouse/ClickHouse/pull/28028) ([Dmitry Novik](https://github.com/novikd)). +* Support ON CONFLICT clause when inserting into PostgreSQL table engine or table function. Closes [#27727](https://github.com/ClickHouse/ClickHouse/issues/27727). [#28081](https://github.com/ClickHouse/ClickHouse/pull/28081) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Support implicit conversions between index in operator `[]` and key of type `Map` (e.g. different `Int` types, `String` and `FixedString`). [#28096](https://github.com/ClickHouse/ClickHouse/pull/28096) ([Anton Popov](https://github.com/CurtizJ)). +* Enable optimize_distributed_group_by_sharding_key by default. [#28105](https://github.com/ClickHouse/ClickHouse/pull/28105) ([Azat Khuzhin](https://github.com/azat)). +* Fix `zookeeper_log.address` (before the first patch in this PR the address was always `::`) and reduce number of calls `getpeername(2)` for this column (since each time entry for `zookeeper_log` is added `getpeername()` is called, cache this address in the zookeeper client to avoid this). [#28212](https://github.com/ClickHouse/ClickHouse/pull/28212) ([Azat Khuzhin](https://github.com/azat)). +* Fix removing of parts in a Temporary state which can lead to an unexpected exception (`Part %name% doesn't exist`). Fixes [#23661](https://github.com/ClickHouse/ClickHouse/issues/23661). [#28221](https://github.com/ClickHouse/ClickHouse/pull/28221) ([Azat Khuzhin](https://github.com/azat)). +* Added libhdfs3_conf in server config instead of export env LIBHDFS3_CONF in clickhouse-server.service. [#28268](https://github.com/ClickHouse/ClickHouse/pull/28268) ([Zhichang Yu](https://github.com/yuzhichang)). +* Use real tmp file instead of predefined "rows_sources" for vertical merges. This avoids generating garbage directories in tmp disks. [#28299](https://github.com/ClickHouse/ClickHouse/pull/28299) ([Amos Bird](https://github.com/amosbird)). +* Speed up data parts loading by delaying table startup process. [#28313](https://github.com/ClickHouse/ClickHouse/pull/28313) ([Amos Bird](https://github.com/amosbird)). +* Allow ssl connection for RabbitMQ engine. [#28365](https://github.com/ClickHouse/ClickHouse/pull/28365) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix removing of parts in a Temporary state (follow up for [#28221](https://github.com/ClickHouse/ClickHouse/issues/28221)). [#28366](https://github.com/ClickHouse/ClickHouse/pull/28366) ([Azat Khuzhin](https://github.com/azat)). +* Do not allow creating StorageMaterializedPostgreSQL with bad arguments. Closes [#28423](https://github.com/ClickHouse/ClickHouse/issues/28423). [#28430](https://github.com/ClickHouse/ClickHouse/pull/28430) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Introduce `connection_wait_timeout` (default to 5 seconds, 0 - do not wait) setting for MySQL engine. [#28474](https://github.com/ClickHouse/ClickHouse/pull/28474) ([Azat Khuzhin](https://github.com/azat)). +* Fix strange sessions expiration logic in Keeper. Probably it should help in CI: https://clickhouse-test-reports.s3.yandex.net/0/6bd9b82141c98dcd7796fd9d08326831095ba519/stress_test_(debug).html#fail1. [#28519](https://github.com/ClickHouse/ClickHouse/pull/28519) ([alesapin](https://github.com/alesapin)). +* To be added. Closes [#28529](https://github.com/ClickHouse/ClickHouse/issues/28529). [#28614](https://github.com/ClickHouse/ClickHouse/pull/28614) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Bug Fix +* Bugfix for windowFunnel's "strict" mode. This fixes [#27469](https://github.com/ClickHouse/ClickHouse/issues/27469). [#27563](https://github.com/ClickHouse/ClickHouse/pull/27563) ([achimbab](https://github.com/achimbab)). +* - Fix bug with aliased column in `Distributed` table. [#27652](https://github.com/ClickHouse/ClickHouse/pull/27652) ([Vladimir C](https://github.com/vdimir)). +* Fixed another case of `Unexpected merged part ... intersecting drop range ...` error. [#27656](https://github.com/ClickHouse/ClickHouse/pull/27656) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix bad type cast when functions like `arrayHas` are applied to arrays of LowCardinality of Nullable of different non-numeric types like `DateTime` and `DateTime64`. In previous versions bad cast occurs. In new version it will lead to exception. This closes [#26330](https://github.com/ClickHouse/ClickHouse/issues/26330). [#27682](https://github.com/ClickHouse/ClickHouse/pull/27682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix column filtering with union distinct in subquery. Closes [#27578](https://github.com/ClickHouse/ClickHouse/issues/27578). [#27689](https://github.com/ClickHouse/ClickHouse/pull/27689) ([Kseniia Sumarokova](https://github.com/kssenii)). +* After https://github.com/ClickHouse/ClickHouse/pull/26384. To execute `GRANT WITH REPLACE OPTION` now the current user should have `GRANT OPTION` for access rights it's going to grant AND for access rights it's going to revoke. [#27701](https://github.com/ClickHouse/ClickHouse/pull/27701) ([Vitaly Baranov](https://github.com/vitlibar)). +* After https://github.com/ClickHouse/ClickHouse/pull/25687. Add backquotes for the default database shown in CREATE USER. [#27702](https://github.com/ClickHouse/ClickHouse/pull/27702) ([Vitaly Baranov](https://github.com/vitlibar)). +* Remove duplicated source files in CMakeLists.txt in arrow-cmake. [#27736](https://github.com/ClickHouse/ClickHouse/pull/27736) ([李扬](https://github.com/taiyang-li)). +* Fix possible crash when asynchronous connection draining is enabled and hedged connection is disabled. [#27774](https://github.com/ClickHouse/ClickHouse/pull/27774) ([Amos Bird](https://github.com/amosbird)). +* Prevent crashes for some formats when NULL (tombstone) message was coming from Kafka. Closes [#19255](https://github.com/ClickHouse/ClickHouse/issues/19255). [#27794](https://github.com/ClickHouse/ClickHouse/pull/27794) ([filimonov](https://github.com/filimonov)). +* Fix a rare bug in `DROP PART` which can lead to the error `Unexpected merged part intersects drop range`. [#27807](https://github.com/ClickHouse/ClickHouse/pull/27807) ([alesapin](https://github.com/alesapin)). +* Fix a couple of bugs that may cause replicas to diverge. [#27808](https://github.com/ClickHouse/ClickHouse/pull/27808) ([Alexander Tokmakov](https://github.com/tavplubix)). +* After https://github.com/ClickHouse/ClickHouse/pull/26864. Fix shutdown of `NamedSessionStorage`: session contexts stored in `NamedSessionStorage` are now destroyed before destroying the global context. [#27875](https://github.com/ClickHouse/ClickHouse/pull/27875) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix PostgreSQL-style cast (`::` operator) with negative numbers. [#27876](https://github.com/ClickHouse/ClickHouse/pull/27876) ([Anton Popov](https://github.com/CurtizJ)). +* Fix selecting with extremes from a column of the type `LowCardinality(UUID)`. [#27918](https://github.com/ClickHouse/ClickHouse/pull/27918) ([Vitaly Baranov](https://github.com/vitlibar)). +* Check cluster name before creating Distributed table, do not allow to create a table with incorrect cluster name. Fixes [#27832](https://github.com/ClickHouse/ClickHouse/issues/27832). [#27927](https://github.com/ClickHouse/ClickHouse/pull/27927) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix checking access grants when executing GRANT WITH REPLACE statement with ON CLUSTER clause. This PR improves fix https://github.com/ClickHouse/ClickHouse/pull/27701. [#27983](https://github.com/ClickHouse/ClickHouse/pull/27983) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix cases, when read buffer fails with 'attempt to read after end of file'. Closes [#26149](https://github.com/ClickHouse/ClickHouse/issues/26149). [#28150](https://github.com/ClickHouse/ClickHouse/pull/28150) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + +#### Build/Testing/Packaging Improvement +* Enable Thread Fuzzer in Stress Test. Thread Fuzzer is ClickHouse feature that allows to test more permutations of thread scheduling and discover more potential issues. This closes [#9813](https://github.com/ClickHouse/ClickHouse/issues/9813). This closes [#9814](https://github.com/ClickHouse/ClickHouse/issues/9814). This closes [#9515](https://github.com/ClickHouse/ClickHouse/issues/9515). This closes [#9516](https://github.com/ClickHouse/ClickHouse/issues/9516). [#27538](https://github.com/ClickHouse/ClickHouse/pull/27538) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add support for build with `clang-13`. This closes [#27705](https://github.com/ClickHouse/ClickHouse/issues/27705). [#27714](https://github.com/ClickHouse/ClickHouse/pull/27714) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve support for build with `clang-13`. [#27777](https://github.com/ClickHouse/ClickHouse/pull/27777) ([Sergei Semin](https://github.com/syominsergey)). +* Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). +* Print out git status information at CMake configure stage. [#28047](https://github.com/ClickHouse/ClickHouse/pull/28047) ([Braulio Valdivielso Martínez](https://github.com/BraulioVM)). +* Add new log level `` for testing environments. [#28559](https://github.com/ClickHouse/ClickHouse/pull/28559) ([alesapin](https://github.com/alesapin)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)). +* Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). +* Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). +* Fix bug in clickhouse-keeper which can lead to endless logs when `rotate_logs_interval` decreased. [#28152](https://github.com/ClickHouse/ClickHouse/pull/28152) ([alesapin](https://github.com/alesapin)). +* Multiple small fixes for projections. See detailed description in pr. [#28178](https://github.com/ClickHouse/ClickHouse/pull/28178) ([Amos Bird](https://github.com/amosbird)). +* Fix incorrect behavior in `clickhouse-keeper` when list watches (`getChildren`) triggered with `set` requests for children. [#28190](https://github.com/ClickHouse/ClickHouse/pull/28190) ([alesapin](https://github.com/alesapin)). +* Fix a rare bug in `clickhouse-keeper` when the client can receive a watch response before request-response. [#28197](https://github.com/ClickHouse/ClickHouse/pull/28197) ([alesapin](https://github.com/alesapin)). +* Fix possible read of uninitialized memory for queries with `Nullable(LowCardinality)` type and extremes. Fixes [#28165](https://github.com/ClickHouse/ClickHouse/issues/28165). [#28205](https://github.com/ClickHouse/ClickHouse/pull/28205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix reading of custom TLD w/o new line at EOF. [#28213](https://github.com/ClickHouse/ClickHouse/pull/28213) ([Azat Khuzhin](https://github.com/azat)). +* Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +* Fix intersecting parts due to new part had been replaced with an empty part. [#28310](https://github.com/ClickHouse/ClickHouse/pull/28310) ([Azat Khuzhin](https://github.com/azat)). +* Fix NOT-IN index optimization when not all key columns are used. This fixes [#28120](https://github.com/ClickHouse/ClickHouse/issues/28120). [#28315](https://github.com/ClickHouse/ClickHouse/pull/28315) ([Amos Bird](https://github.com/amosbird)). +* Fix non joined rows from nullable column. Close [#27691](https://github.com/ClickHouse/ClickHouse/issues/27691). [#28349](https://github.com/ClickHouse/ClickHouse/pull/28349) ([Vladimir C](https://github.com/vdimir)). +* Fix rare case when changes of `clickhouse-keeper` settings may lead to lost logs and server hung. [#28360](https://github.com/ClickHouse/ClickHouse/pull/28360) ([alesapin](https://github.com/alesapin)). +* Fix lack of quotes for table names in MaterializedPostgreSQL engine. Closes [#28316](https://github.com/ClickHouse/ClickHouse/issues/28316). [#28433](https://github.com/ClickHouse/ClickHouse/pull/28433) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed possible ZooKeeper watches leak on background processing of distributed DDL queue. Closes [#26036](https://github.com/ClickHouse/ClickHouse/issues/26036). [#28446](https://github.com/ClickHouse/ClickHouse/pull/28446) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)). +* Fix `There is no subcolumn` error, while select from tables, which have `Nested` columns and scalar columns with dot in name and the same prefix as `Nested` (e.g. `n.id UInt32, n.arr1 Array(UInt64), n.arr2 Array(UInt64)`). [#28531](https://github.com/ClickHouse/ClickHouse/pull/28531) ([Anton Popov](https://github.com/CurtizJ)). +* Fix UUID overlap in DROP TABLE for internal DDL from MaterializeMySQL. [#28533](https://github.com/ClickHouse/ClickHouse/pull/28533) ([Azat Khuzhin](https://github.com/azat)). +* Fix endless loop for truncated bzip2 archive. [#28543](https://github.com/ClickHouse/ClickHouse/pull/28543) ([Azat Khuzhin](https://github.com/azat)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Add executable table function'. [#23192](https://github.com/ClickHouse/ClickHouse/pull/23192) ([ruct](https://github.com/ruct)). +* NO CL ENTRY: 'DOCSUP-12413: macros support in functions cluster and clusterAllReplicas'. [#27759](https://github.com/ClickHouse/ClickHouse/pull/27759) ([olgarev](https://github.com/olgarev)). +* NO CL ENTRY: 'Revert "less sys calls #2: make vdso work again"'. [#27829](https://github.com/ClickHouse/ClickHouse/pull/27829) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Do not miss exceptions from the ThreadPool"'. [#27844](https://github.com/ClickHouse/ClickHouse/pull/27844) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Improve 01730_distributed_group_by_no_merge_order_by_long"'. [#28128](https://github.com/ClickHouse/ClickHouse/pull/28128) ([alesapin](https://github.com/alesapin)). +* NO CL ENTRY: 'Revert "Revert "less sys calls #2: make vdso work again""'. [#28132](https://github.com/ClickHouse/ClickHouse/pull/28132) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Update src/Functions/GatherUtils/Sources.h'. [#28186](https://github.com/ClickHouse/ClickHouse/pull/28186) ([sdk2](https://github.com/sdk2)). +* NO CL ENTRY: 'Revert "Add test for [#13398](https://github.com/ClickHouse/ClickHouse/issues/13398)"'. [#28274](https://github.com/ClickHouse/ClickHouse/pull/28274) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'fix minor typo'. [#28629](https://github.com/ClickHouse/ClickHouse/pull/28629) ([flynn](https://github.com/ucasfl)). + diff --git a/docs/changelogs/v21.10.2.15-stable.md b/docs/changelogs/v21.10.2.15-stable.md new file mode 100644 index 00000000000..05e278f03ae --- /dev/null +++ b/docs/changelogs/v21.10.2.15-stable.md @@ -0,0 +1,65 @@ +### ClickHouse release v21.10.2.15-stable FIXME as compared to v21.10.1.8013-prestable + +#### Improvement +* Backported in [#29944](https://github.com/ClickHouse/ClickHouse/issues/29944): Update zoneinfo files to 2021c. [#29925](https://github.com/ClickHouse/ClickHouse/pull/29925) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#29776](https://github.com/ClickHouse/ClickHouse/issues/29776): Allow using a materialized column as the sharding key in a distributed table even if `insert_allow_materialized_columns=0`:. [#28637](https://github.com/ClickHouse/ClickHouse/pull/28637) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#28792](https://github.com/ClickHouse/ClickHouse/issues/28792): Fix transformation of disjunctions chain to `IN` (controlled by settings `optimize_min_equality_disjunction_chain_length`) in distributed queries with settings `legacy_column_name_of_tuple_literal = 0`. [#28658](https://github.com/ClickHouse/ClickHouse/pull/28658) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#29127](https://github.com/ClickHouse/ClickHouse/issues/29127): Fix bug in `clickhouse-keeper-converter` which can lead to incorrect ZooKeeper log deserialization. [#29071](https://github.com/ClickHouse/ClickHouse/pull/29071) ([小路](https://github.com/nicelulu)). +* Backported in [#29970](https://github.com/ClickHouse/ClickHouse/issues/29970): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#30051](https://github.com/ClickHouse/ClickHouse/issues/30051): Fix releasing query ID and session ID at the end of query processing while handing gRPC call. This PR fixes flaky test [test_grpc_protocol/test.py::test_session](https://clickhouse-test-reports.s3.yandex.net/0/1ac03811a2df9717fa7c633d1af03def821d24b6/integration_tests_(memory).html). [#29954](https://github.com/ClickHouse/ClickHouse/pull/29954) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#29054](https://github.com/ClickHouse/ClickHouse/issues/29054): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#28795](https://github.com/ClickHouse/ClickHouse/issues/28795): - Fix the number of arguments required by s2RectAdd and s2RectContains functions. [#28663](https://github.com/ClickHouse/ClickHouse/pull/28663) ([Bharat Nallan](https://github.com/bharatnc)). +* Backported in [#28794](https://github.com/ClickHouse/ClickHouse/issues/28794): Add Settings.Names, Settings.Values aliases for system.processes table. [#28685](https://github.com/ClickHouse/ClickHouse/pull/28685) ([Vitaly Orlov](https://github.com/orloffv)). +* Backported in [#28793](https://github.com/ClickHouse/ClickHouse/issues/28793): Fix the coredump in the creation of distributed tables, when the parameters passed in are wrong. [#28686](https://github.com/ClickHouse/ClickHouse/pull/28686) ([Zhiyong Wang](https://github.com/ljcui)). +* Backported in [#28812](https://github.com/ClickHouse/ClickHouse/issues/28812): Fix possible crash for `SELECT` with partially created aggregate projection in case of exception. [#28700](https://github.com/ClickHouse/ClickHouse/pull/28700) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#28790](https://github.com/ClickHouse/ClickHouse/issues/28790): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)). +* Backported in [#29150](https://github.com/ClickHouse/ClickHouse/issues/29150): Fix extremely rare case when ReplicatedMergeTree replicas can diverge after hard reboot of all replicas. The error looks like `Part ... intersects (previous|next) part ...`. [#28817](https://github.com/ClickHouse/ClickHouse/pull/28817) ([alesapin](https://github.com/alesapin)). +* Backported in [#28843](https://github.com/ClickHouse/ClickHouse/issues/28843): Fix expressions compilation with short circuit evaluation. [#28821](https://github.com/ClickHouse/ClickHouse/pull/28821) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#28992](https://github.com/ClickHouse/ClickHouse/issues/28992): Fixed a race condition between `DROP PART` and `REPLACE/MOVE PARTITION` that might cause replicas to diverge in rare cases. [#28864](https://github.com/ClickHouse/ClickHouse/pull/28864) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#28950](https://github.com/ClickHouse/ClickHouse/issues/28950): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28926](https://github.com/ClickHouse/ClickHouse/issues/28926): Fix bug with LowCardinality in short-curcuit function evaluation. Closes [#28884](https://github.com/ClickHouse/ClickHouse/issues/28884). [#28887](https://github.com/ClickHouse/ClickHouse/pull/28887) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#29162](https://github.com/ClickHouse/ClickHouse/issues/29162): Fix queries to external databases (i.e. MySQL) with multiple columns in IN ( i.e. `(k,v) IN ((1, 2))` ) (but note that this has some backward incompatibility for the `clickhouse-copier` since it uses alias for tuple element). [#28888](https://github.com/ClickHouse/ClickHouse/pull/28888) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29108](https://github.com/ClickHouse/ClickHouse/issues/29108): Fix waiting for mutation with `mutations_sync=2`. [#28889](https://github.com/ClickHouse/ClickHouse/pull/28889) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#28928](https://github.com/ClickHouse/ClickHouse/issues/28928): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#28951](https://github.com/ClickHouse/ClickHouse/issues/28951): Fix bad optimizations of ORDER BY if it contains WITH FILL. This closes [#28908](https://github.com/ClickHouse/ClickHouse/issues/28908). This closes [#26049](https://github.com/ClickHouse/ClickHouse/issues/26049). [#28910](https://github.com/ClickHouse/ClickHouse/pull/28910) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#29023](https://github.com/ClickHouse/ClickHouse/issues/29023): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29814](https://github.com/ClickHouse/ClickHouse/issues/29814): Do not allow to reuse previous credentials in case of inter-server secret (Before INSERT via Buffer/Kafka to Distributed table with interserver secret configured for that cluster, may re-use previously set user for that connection). [#29060](https://github.com/ClickHouse/ClickHouse/pull/29060) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29192](https://github.com/ClickHouse/ClickHouse/issues/29192): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#29400](https://github.com/ClickHouse/ClickHouse/issues/29400): Send normal `Database doesn't exist error` (`UNKNOWN_DATABASE`) to the client (via TCP) instead of `Attempt to read after eof` (`ATTEMPT_TO_READ_AFTER_EOF`). [#29229](https://github.com/ClickHouse/ClickHouse/pull/29229) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29358](https://github.com/ClickHouse/ClickHouse/issues/29358): Fix possible `Table columns structure in ZooKeeper is different from local table structure` exception while recreating or creating new replicas of `ReplicatedMergeTree`, when one of table columns have default expressions with case-insensitive functions. [#29266](https://github.com/ClickHouse/ClickHouse/pull/29266) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#29450](https://github.com/ClickHouse/ClickHouse/issues/29450): Fix failed assertion in ReadBufferFromHDFS. Update libhdfs3 library to be able to run in tests in debug. Closes [#29251](https://github.com/ClickHouse/ClickHouse/issues/29251). Closes [#27814](https://github.com/ClickHouse/ClickHouse/issues/27814). [#29276](https://github.com/ClickHouse/ClickHouse/pull/29276) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29302](https://github.com/ClickHouse/ClickHouse/issues/29302): Fix connection timeouts (`send_timeout`/`receive_timeout`). [#29282](https://github.com/ClickHouse/ClickHouse/pull/29282) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29382](https://github.com/ClickHouse/ClickHouse/issues/29382): Remove window function `nth_value` as it is not memory-safe. This closes [#29347](https://github.com/ClickHouse/ClickHouse/issues/29347). [#29348](https://github.com/ClickHouse/ClickHouse/pull/29348) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#29439](https://github.com/ClickHouse/ClickHouse/issues/29439): Fix replicated access storage not shutting down cleanly when misconfigured. [#29388](https://github.com/ClickHouse/ClickHouse/pull/29388) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Backported in [#29488](https://github.com/ClickHouse/ClickHouse/issues/29488): Fix Logical error `Cannot capture columns` in functions greatest/least. Closes [#29334](https://github.com/ClickHouse/ClickHouse/issues/29334). [#29454](https://github.com/ClickHouse/ClickHouse/pull/29454) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#29537](https://github.com/ClickHouse/ClickHouse/issues/29537): Fix possible `Block structure mismatch` for subqueries with pushed-down `HAVING` predicate. Fixes [#29010](https://github.com/ClickHouse/ClickHouse/issues/29010). [#29475](https://github.com/ClickHouse/ClickHouse/pull/29475) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29590](https://github.com/ClickHouse/ClickHouse/issues/29590): In ODBC bridge add retries for error Invalid cursor state. It is a retriable error. Closes [#29473](https://github.com/ClickHouse/ClickHouse/issues/29473). [#29518](https://github.com/ClickHouse/ClickHouse/pull/29518) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29571](https://github.com/ClickHouse/ClickHouse/issues/29571): Fix bug in check `pathStartsWith` becuase there was bug with the usage of `std::mismatch`: ` The behavior is undefined if the second range is shorter than the first range.`. [#29531](https://github.com/ClickHouse/ClickHouse/pull/29531) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29725](https://github.com/ClickHouse/ClickHouse/issues/29725): Fix null deference for `GROUP BY WITH TOTALS HAVING` (when the column from `HAVING` wasn't selected). [#29553](https://github.com/ClickHouse/ClickHouse/pull/29553) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29629](https://github.com/ClickHouse/ClickHouse/issues/29629): Fix rare segfault in `ALTER MODIFY` query when using incorrect table identifier in `DEFAULT` expression like `x.y.z...` Fixes [#29184](https://github.com/ClickHouse/ClickHouse/issues/29184). [#29573](https://github.com/ClickHouse/ClickHouse/pull/29573) ([alesapin](https://github.com/alesapin)). +* Backported in [#29657](https://github.com/ClickHouse/ClickHouse/issues/29657): Fix JIT expression compilation with aliases and short-circuit expression evaluation. Closes [#29403](https://github.com/ClickHouse/ClickHouse/issues/29403). [#29574](https://github.com/ClickHouse/ClickHouse/pull/29574) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#29749](https://github.com/ClickHouse/ClickHouse/issues/29749): Condition in filter predicate could be lost after push-down optimisation. [#29625](https://github.com/ClickHouse/ClickHouse/pull/29625) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29850](https://github.com/ClickHouse/ClickHouse/issues/29850): Fix concurrent access to `LowCardinality` during `GROUP BY` (leads to SIGSEGV). [#29782](https://github.com/ClickHouse/ClickHouse/pull/29782) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29910](https://github.com/ClickHouse/ClickHouse/issues/29910): Fix bad cast in `ATTACH TABLE ... FROM 'path'` query when non-string literal is used instead of path. It may lead to reading of uninitialized memory. [#29790](https://github.com/ClickHouse/ClickHouse/pull/29790) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#29866](https://github.com/ClickHouse/ClickHouse/issues/29866): Avoid `Timeout exceeded: elapsed 18446744073.709553 seconds` error that might happen in extremely rare cases, presumably due to some bug in kernel. Fixes [#29154](https://github.com/ClickHouse/ClickHouse/issues/29154). [#29811](https://github.com/ClickHouse/ClickHouse/pull/29811) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30023](https://github.com/ClickHouse/ClickHouse/issues/30023): MaterializedMySQL: Fix an issue where if the connection to MySQL was lost, only parts of a transaction could be processed. [#29837](https://github.com/ClickHouse/ClickHouse/pull/29837) ([Håvard Kvålen](https://github.com/havardk)). +* Backported in [#29876](https://github.com/ClickHouse/ClickHouse/issues/29876): Fix system tables recreation check (fails to detect changes in enum values). [#29857](https://github.com/ClickHouse/ClickHouse/pull/29857) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30058](https://github.com/ClickHouse/ClickHouse/issues/30058): Fix potential resource leak of the concurrent query limit of merge tree tables introduced in https://github.com/ClickHouse/ClickHouse/pull/19544 . [#29879](https://github.com/ClickHouse/ClickHouse/pull/29879) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#30208](https://github.com/ClickHouse/ClickHouse/issues/30208): Fix data-race between `LogSink::writeMarks()` and `LogSource` in `StorageLog`. [#29946](https://github.com/ClickHouse/ClickHouse/pull/29946) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30213](https://github.com/ClickHouse/ClickHouse/issues/30213): Fix possible data-race between `FileChecker` and `StorageLog`/`StorageStripeLog`. [#29959](https://github.com/ClickHouse/ClickHouse/pull/29959) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30067](https://github.com/ClickHouse/ClickHouse/issues/30067): Fix crash of sample by `tuple()`, closes [#30004](https://github.com/ClickHouse/ClickHouse/issues/30004). [#30016](https://github.com/ClickHouse/ClickHouse/pull/30016) ([flynn](https://github.com/ucasfl)). +* Backported in [#30127](https://github.com/ClickHouse/ClickHouse/issues/30127): Dropped `Memory` database might reappear after server restart, it's fixed ([#29795](https://github.com/ClickHouse/ClickHouse/issues/29795)). Also added `force_remove_data_recursively_on_drop` setting as a workaround for `Directory not empty` error when dropping `Ordinary` database (because it's not possible to remove data leftovers manually in cloud environment). [#30054](https://github.com/ClickHouse/ClickHouse/pull/30054) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30232](https://github.com/ClickHouse/ClickHouse/issues/30232): Fix INSERT SELECT incorrectly fills MATERIALIZED column based of Nullable column. [#30189](https://github.com/ClickHouse/ClickHouse/pull/30189) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30264](https://github.com/ClickHouse/ClickHouse/issues/30264): FlatDictionary, HashedDictionary fix bytes_allocated calculation for nullable attributes. [#30238](https://github.com/ClickHouse/ClickHouse/pull/30238) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#30307](https://github.com/ClickHouse/ClickHouse/issues/30307): Fix crash with shortcircuit and lowcardinality in multiIf. [#30243](https://github.com/ClickHouse/ClickHouse/pull/30243) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#30291](https://github.com/ClickHouse/ClickHouse/issues/30291): Fix ComplexKeyHashedDictionary, ComplexKeySparseHashedDictionary parsing `preallocate` option from layout config. [#30246](https://github.com/ClickHouse/ClickHouse/pull/30246) ([Maksim Kita](https://github.com/kitaisreal)). + +#### NO CL CATEGORY + +* Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30182](https://github.com/ClickHouse/ClickHouse/pull/30182) ([Raúl Marín](https://github.com/Algunenano)). + diff --git a/docs/changelogs/v21.10.3.9-stable.md b/docs/changelogs/v21.10.3.9-stable.md new file mode 100644 index 00000000000..78240367d55 --- /dev/null +++ b/docs/changelogs/v21.10.3.9-stable.md @@ -0,0 +1,45 @@ +### ClickHouse release v21.10.3.9-stable FIXME as compared to v21.10.2.15-stable + +#### New Feature +* Backported in [#30712](https://github.com/ClickHouse/ClickHouse/issues/30712): CompiledExpressionCache limit elements size using `compiled_expression_cache_elements_size` setting. [#30667](https://github.com/ClickHouse/ClickHouse/pull/30667) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Improvement +* Backported in [#30453](https://github.com/ClickHouse/ClickHouse/issues/30453): Allow symlinks to files in user_files directory for file table function. [#30309](https://github.com/ClickHouse/ClickHouse/pull/30309) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#30635](https://github.com/ClickHouse/ClickHouse/issues/30635): More full support of positional arguments. [#30433](https://github.com/ClickHouse/ClickHouse/pull/30433) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Bug Fix +* Backported in [#30620](https://github.com/ClickHouse/ClickHouse/issues/30620): Fix reading from empty file on encrypted disk. [#30494](https://github.com/ClickHouse/ClickHouse/pull/30494) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#31369](https://github.com/ClickHouse/ClickHouse/issues/31369): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release + +* Backported in [#30915](https://github.com/ClickHouse/ClickHouse/issues/30915): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#30824](https://github.com/ClickHouse/ClickHouse/issues/30824): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30766](https://github.com/ClickHouse/ClickHouse/issues/30766): Fix hanging DDL queries on Replicated database while adding a new replica. [#29328](https://github.com/ClickHouse/ClickHouse/pull/29328) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Backported in [#30504](https://github.com/ClickHouse/ClickHouse/issues/30504): Fixed incorrect behaviour of setting `materialized_postgresql_tables_list` at server restart. Found in [#28529](https://github.com/ClickHouse/ClickHouse/issues/28529). [#29686](https://github.com/ClickHouse/ClickHouse/pull/29686) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#30137](https://github.com/ClickHouse/ClickHouse/issues/30137): Fix error `Port is already connected` for queries with `GLOBAL IN` and `WITH TOTALS`. Only for 21.9 and 21.10. [#30086](https://github.com/ClickHouse/ClickHouse/pull/30086) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#30465](https://github.com/ClickHouse/ClickHouse/issues/30465): Support nullable arguments in function `initializeAggregation`. [#30177](https://github.com/ClickHouse/ClickHouse/pull/30177) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#30353](https://github.com/ClickHouse/ClickHouse/issues/30353): Fix `pread_fake_async`/`pread_threadpool` with `min_bytes_to_use_direct_io`. [#30191](https://github.com/ClickHouse/ClickHouse/pull/30191) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30334](https://github.com/ClickHouse/ClickHouse/issues/30334): * Allow identifiers staring with numbers in multiple joins. [#30230](https://github.com/ClickHouse/ClickHouse/pull/30230) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#30654](https://github.com/ClickHouse/ClickHouse/issues/30654): Fix `[I]LIKE` function. Closes [#28661](https://github.com/ClickHouse/ClickHouse/issues/28661). [#30244](https://github.com/ClickHouse/ClickHouse/pull/30244) ([Nikolay Degterinsky](https://github.com/evillique)). +* Backported in [#30378](https://github.com/ClickHouse/ClickHouse/issues/30378): fix replaceRegexpAll bug. [#30292](https://github.com/ClickHouse/ClickHouse/pull/30292) ([Memo](https://github.com/Joeywzr)). +* Backported in [#30526](https://github.com/ClickHouse/ClickHouse/issues/30526): Fixed segfault which might happen if session expired during execution of REPLACE PARTITION. [#30432](https://github.com/ClickHouse/ClickHouse/pull/30432) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30587](https://github.com/ClickHouse/ClickHouse/issues/30587): * Fix deadlock on ALTER with scalar subquery to the same table, close [#30461](https://github.com/ClickHouse/ClickHouse/issues/30461). [#30492](https://github.com/ClickHouse/ClickHouse/pull/30492) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#30608](https://github.com/ClickHouse/ClickHouse/issues/30608): Limit push down optimization could cause a error `Cannot find column`. Fixes [#30438](https://github.com/ClickHouse/ClickHouse/issues/30438). [#30562](https://github.com/ClickHouse/ClickHouse/pull/30562) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#30747](https://github.com/ClickHouse/ClickHouse/issues/30747): Update aws-sdk submodule for throttling in Yandex.S3. [#30646](https://github.com/ClickHouse/ClickHouse/pull/30646) ([ianton-ru](https://github.com/ianton-ru)). +* Backported in [#30751](https://github.com/ClickHouse/ClickHouse/issues/30751): Functions for case-insensitive search in UTF8 strings like `positionCaseInsensitiveUTF8` and `countSubstringsCaseInsensitiveUTF8` might find substrings that actually does not match, it's fixed. [#30663](https://github.com/ClickHouse/ClickHouse/pull/30663) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30709](https://github.com/ClickHouse/ClickHouse/issues/30709): Fix PREWHERE with WHERE in case of always true PREWHERE. [#30668](https://github.com/ClickHouse/ClickHouse/pull/30668) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30771](https://github.com/ClickHouse/ClickHouse/issues/30771): Fixed a race condition between `REPLACE/MOVE PARTITION` and background merge in non-replicated `MergeTree` that might cause a part of moved/replaced data to remain in partition. Fixes [#29327](https://github.com/ClickHouse/ClickHouse/issues/29327). [#30717](https://github.com/ClickHouse/ClickHouse/pull/30717) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30859](https://github.com/ClickHouse/ClickHouse/issues/30859): Fixed ambiguity when extracting auxiliary ZooKeeper name from ZooKeeper path in `ReplicatedMergeTree`. Previously server might fail to start with `Unknown auxiliary ZooKeeper name` if ZooKeeper path contains a colon. Fixes [#29052](https://github.com/ClickHouse/ClickHouse/issues/29052). Also it was allowed to specify ZooKeeper path that does not start with slash, but now it's deprecated and creation of new tables with such path is not allowed. Slashes and colons in auxiliary ZooKeeper names are not allowed too. [#30822](https://github.com/ClickHouse/ClickHouse/pull/30822) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30926](https://github.com/ClickHouse/ClickHouse/issues/30926): Fix set index not used in AND/OR expressions when there are more than two operands. This fixes [#30416](https://github.com/ClickHouse/ClickHouse/issues/30416) . [#30887](https://github.com/ClickHouse/ClickHouse/pull/30887) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#31289](https://github.com/ClickHouse/ClickHouse/issues/31289): Fix some corner cases with intersect/except. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31152](https://github.com/ClickHouse/ClickHouse/issues/31152): Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)). +* Backported in [#31041](https://github.com/ClickHouse/ClickHouse/issues/31041): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31203](https://github.com/ClickHouse/ClickHouse/issues/31203): Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31373](https://github.com/ClickHouse/ClickHouse/issues/31373): Fix StorageMerge with aliases and where (it did not work before at all). Closes [#28802](https://github.com/ClickHouse/ClickHouse/issues/28802). [#31044](https://github.com/ClickHouse/ClickHouse/pull/31044) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31255](https://github.com/ClickHouse/ClickHouse/issues/31255): Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)). +* Backported in [#31436](https://github.com/ClickHouse/ClickHouse/issues/31436): Fix bug with group by and positional arguments. Closes [#31280](https://github.com/ClickHouse/ClickHouse/issues/31280)#issuecomment-968696186. [#31420](https://github.com/ClickHouse/ClickHouse/pull/31420) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.10.4.26-stable.md b/docs/changelogs/v21.10.4.26-stable.md new file mode 100644 index 00000000000..b2089fad0c4 --- /dev/null +++ b/docs/changelogs/v21.10.4.26-stable.md @@ -0,0 +1,25 @@ +### ClickHouse release v21.10.4.26-stable FIXME as compared to v21.10.3.9-stable + +#### Performance Improvement +* Backported in [#31731](https://github.com/ClickHouse/ClickHouse/issues/31731): Improve performance of JSON and XML output formats. [#31673](https://github.com/ClickHouse/ClickHouse/pull/31673) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#31573](https://github.com/ClickHouse/ClickHouse/issues/31573): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#31518](https://github.com/ClickHouse/ClickHouse/issues/31518): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)). +* Backported in [#31554](https://github.com/ClickHouse/ClickHouse/issues/31554): Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#31579](https://github.com/ClickHouse/ClickHouse/issues/31579): * Disable `partial_merge_join_left_table_buffer_bytes` before bug in this optimization is fixed. See [#31009](https://github.com/ClickHouse/ClickHouse/issues/31009)). * Remove redundant option `partial_merge_join_optimizations`. [#31528](https://github.com/ClickHouse/ClickHouse/pull/31528) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#31745](https://github.com/ClickHouse/ClickHouse/issues/31745): `RENAME TABLE` query worked incorrectly on attempt to rename an DDL dictionary in `Ordinary` database, it's fixed. [#31638](https://github.com/ClickHouse/ClickHouse/pull/31638) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31794](https://github.com/ClickHouse/ClickHouse/issues/31794): Settings `input_format_allow_errors_num` and `input_format_allow_errors_ratio` did not work for parsing of domain types, such as `IPv4`, it's fixed. Fixes [#31686](https://github.com/ClickHouse/ClickHouse/issues/31686). [#31697](https://github.com/ClickHouse/ClickHouse/pull/31697) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31830](https://github.com/ClickHouse/ClickHouse/issues/31830): Fixed `there are no such cluster here` error on execution of `ON CLUSTER` query if specified cluster name is name of `Replicated` database. [#31723](https://github.com/ClickHouse/ClickHouse/pull/31723) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31815](https://github.com/ClickHouse/ClickHouse/issues/31815): Fix race in JSONEachRowWithProgress output format when data and lines with progress are mixed in output. [#31736](https://github.com/ClickHouse/ClickHouse/pull/31736) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#32016](https://github.com/ClickHouse/ClickHouse/issues/32016): Fix group by / order by / limit by aliases with positional arguments enabled. Closes [#31173](https://github.com/ClickHouse/ClickHouse/issues/31173). [#31741](https://github.com/ClickHouse/ClickHouse/pull/31741) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31762](https://github.com/ClickHouse/ClickHouse/issues/31762): Fix usage of `Buffer` table engine with type `Map`. Fixes [#30546](https://github.com/ClickHouse/ClickHouse/issues/30546). [#31742](https://github.com/ClickHouse/ClickHouse/pull/31742) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#31892](https://github.com/ClickHouse/ClickHouse/issues/31892): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#32031](https://github.com/ClickHouse/ClickHouse/issues/32031): Fix invalid cast of nullable type when nullable primary key is used. This fixes [#31075](https://github.com/ClickHouse/ClickHouse/issues/31075). [#31823](https://github.com/ClickHouse/ClickHouse/pull/31823) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#32074](https://github.com/ClickHouse/ClickHouse/issues/32074): Fix a bug about function transform with decimal args. [#31839](https://github.com/ClickHouse/ClickHouse/pull/31839) ([Shuai li](https://github.com/loneylee)). +* Backported in [#31939](https://github.com/ClickHouse/ClickHouse/issues/31939): - Change configuration path from `keeper_server.session_timeout_ms` to `keeper_server.coordination_settings.session_timeout_ms` when constructing a `KeeperTCPHandler` - Same with `operation_timeout`. [#31859](https://github.com/ClickHouse/ClickHouse/pull/31859) ([JackyWoo](https://github.com/JackyWoo)). +* Backported in [#31909](https://github.com/ClickHouse/ClickHouse/issues/31909): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.10.5.3-stable.md b/docs/changelogs/v21.10.5.3-stable.md new file mode 100644 index 00000000000..a591c4a07a8 --- /dev/null +++ b/docs/changelogs/v21.10.5.3-stable.md @@ -0,0 +1,15 @@ +### ClickHouse release v21.10.5.3-stable FIXME as compared to v21.10.4.26-stable + +#### Bug Fix +* Backported in [#32252](https://github.com/ClickHouse/ClickHouse/issues/32252): Fix skipping columns while writing protobuf. This PR fixes [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160), see the comment [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160)#issuecomment-980595318. [#31988](https://github.com/ClickHouse/ClickHouse/pull/31988) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#32346](https://github.com/ClickHouse/ClickHouse/issues/32346): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#32151](https://github.com/ClickHouse/ClickHouse/issues/32151): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#32093](https://github.com/ClickHouse/ClickHouse/issues/32093): Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32147](https://github.com/ClickHouse/ClickHouse/issues/32147): Fixed `Directory ... already exists and is not empty` error when detaching part. [#32063](https://github.com/ClickHouse/ClickHouse/pull/32063) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32216](https://github.com/ClickHouse/ClickHouse/issues/32216): Number of active replicas might be determined incorrectly when inserting with quorum if setting `replicated_can_become_leader` is disabled on some replicas. It's fixed. [#32157](https://github.com/ClickHouse/ClickHouse/pull/32157) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32352](https://github.com/ClickHouse/ClickHouse/issues/32352): Fixed crash with SIGFPE in aggregate function `avgWeighted` with `Decimal` argument. Fixes [#32053](https://github.com/ClickHouse/ClickHouse/issues/32053). [#32303](https://github.com/ClickHouse/ClickHouse/pull/32303) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32393](https://github.com/ClickHouse/ClickHouse/issues/32393): Fix `ALTER ... MATERIALIZE COLUMN ...` queries in case when data type of default expression is not equal to the data type of column. [#32348](https://github.com/ClickHouse/ClickHouse/pull/32348) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.10.6.2-stable.md b/docs/changelogs/v21.10.6.2-stable.md new file mode 100644 index 00000000000..da146ee364d --- /dev/null +++ b/docs/changelogs/v21.10.6.2-stable.md @@ -0,0 +1,22 @@ +### ClickHouse release v21.10.6.2-stable FIXME as compared to v21.10.5.3-stable + +#### Bug Fix +* Backported in [#32692](https://github.com/ClickHouse/ClickHouse/issues/32692): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#32680](https://github.com/ClickHouse/ClickHouse/issues/32680): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#32285](https://github.com/ClickHouse/ClickHouse/issues/32285): Dictionaries fix cases when `{condition}` does not work for custom database queries. [#32117](https://github.com/ClickHouse/ClickHouse/pull/32117) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#32730](https://github.com/ClickHouse/ClickHouse/issues/32730): Fix 'APPLY lambda' parsing which could lead to client/server crash. [#32138](https://github.com/ClickHouse/ClickHouse/pull/32138) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#32313](https://github.com/ClickHouse/ClickHouse/issues/32313): XML dictionaries identifiers, used in table create query, can be qualified to `default_database` during upgrade to newer version. Closes [#31963](https://github.com/ClickHouse/ClickHouse/issues/31963). [#32187](https://github.com/ClickHouse/ClickHouse/pull/32187) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#32539](https://github.com/ClickHouse/ClickHouse/issues/32539): Some replication queue entries might hang for `temporary_directories_lifetime` (1 day by default) with `Directory tmp_merge_` or `Part ... (state Deleting) already exists, but it will be deleted soon` or similar error. It's fixed. Fixes [#29616](https://github.com/ClickHouse/ClickHouse/issues/29616). [#32201](https://github.com/ClickHouse/ClickHouse/pull/32201) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32709](https://github.com/ClickHouse/ClickHouse/issues/32709): Fix failures in queries that are trying to use skipping indices, which are not materialized yet. Fixes [#32292](https://github.com/ClickHouse/ClickHouse/issues/32292) and [#30343](https://github.com/ClickHouse/ClickHouse/issues/30343). [#32359](https://github.com/ClickHouse/ClickHouse/pull/32359) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#32567](https://github.com/ClickHouse/ClickHouse/issues/32567): Fix crash in `JoinCommon::removeColumnNullability`, close [#32458](https://github.com/ClickHouse/ClickHouse/issues/32458). [#32508](https://github.com/ClickHouse/ClickHouse/pull/32508) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#32794](https://github.com/ClickHouse/ClickHouse/issues/32794): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). +* Backported in [#33182](https://github.com/ClickHouse/ClickHouse/issues/33182): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#33655](https://github.com/ClickHouse/ClickHouse/issues/33655): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release): + +* Backported in [#32657](https://github.com/ClickHouse/ClickHouse/issues/32657): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.1.8636-prestable.md b/docs/changelogs/v21.11.1.8636-prestable.md new file mode 100644 index 00000000000..ade8084055c --- /dev/null +++ b/docs/changelogs/v21.11.1.8636-prestable.md @@ -0,0 +1,196 @@ +### ClickHouse release v21.11.1.8636-prestable FIXME as compared to v21.10.1.8013-prestable + +#### Backward Incompatible Change +* Now ClickHouse Keeper snapshots compressed with `ZSTD` codec by default instead of custom ClickHouse LZ4 block compression. This behavior can be turned off with `compress_snapshots_with_zstd_format` coordination setting (must be equal on all quorum replicas). Backward incompatibility is quite rare and may happen only when new node will send snapshot (happens in case of recovery) to the old node which is unable to read snapshots in ZSTD format. [#29417](https://github.com/ClickHouse/ClickHouse/pull/29417) ([alesapin](https://github.com/alesapin)). +* Function `bayesAB` is removed. This closes [#26233](https://github.com/ClickHouse/ClickHouse/issues/26233). [#29934](https://github.com/ClickHouse/ClickHouse/pull/29934) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove MergeTree table setting `write_final_mark`. It will be always `true`. [#30455](https://github.com/ClickHouse/ClickHouse/pull/30455) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Change order of json_path and json in sql json function (to be correct). Closes [#30449](https://github.com/ClickHouse/ClickHouse/issues/30449). [#30474](https://github.com/ClickHouse/ClickHouse/pull/30474) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove setting odbc_max_field_size because it is not used anymore. Closes [#30413](https://github.com/ClickHouse/ClickHouse/issues/30413). [#30778](https://github.com/ClickHouse/ClickHouse/pull/30778) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### New Feature +* Add CapnProto output format, refactor CapnProto input format. [#29291](https://github.com/ClickHouse/ClickHouse/pull/29291) ([Kruglov Pavel](https://github.com/Avogar)). +* clickhouse-format support option `--query`. [#29325](https://github.com/ClickHouse/ClickHouse/pull/29325) ([凌涛](https://github.com/lingtaolf)). +* Users now can set comments to database in `CREATE DATABASE` statement ... [#29429](https://github.com/ClickHouse/ClickHouse/pull/29429) ([Vasily Nemkov](https://github.com/Enmk)). +* New function` mapContainsKeyLike` to get the map that key matches a simple regular expression. [#29471](https://github.com/ClickHouse/ClickHouse/pull/29471) ([凌涛](https://github.com/lingtaolf)). +* Huawei OBS Storage support. Closes [#24294](https://github.com/ClickHouse/ClickHouse/issues/24294). [#29511](https://github.com/ClickHouse/ClickHouse/pull/29511) ([kevin wan](https://github.com/MaxWk)). +* Clickhouse HTTP Server can enable HSTS by set `hsts_max_age` in config.xml with a positive number. [#29516](https://github.com/ClickHouse/ClickHouse/pull/29516) ([凌涛](https://github.com/lingtaolf)). +* - Added MD4 and SHA384 functions. [#29602](https://github.com/ClickHouse/ClickHouse/pull/29602) ([Nikita Tikhomirov](https://github.com/NSTikhomirov)). +* Support EXISTS(subquery). Closes [#6852](https://github.com/ClickHouse/ClickHouse/issues/6852). [#29731](https://github.com/ClickHouse/ClickHouse/pull/29731) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Added function `ngram`. Closes [#29699](https://github.com/ClickHouse/ClickHouse/issues/29699). [#29738](https://github.com/ClickHouse/ClickHouse/pull/29738) ([Maksim Kita](https://github.com/kitaisreal)). +* Returns String with OS Kernel version. [#29755](https://github.com/ClickHouse/ClickHouse/pull/29755) ([Memo](https://github.com/Joeywzr)). +* Predefined configuration for table function remote. Closes [#29756](https://github.com/ClickHouse/ClickHouse/issues/29756). [#29774](https://github.com/ClickHouse/ClickHouse/pull/29774) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add `table` alias to system.tables and `database` alias to system.databases [#29677](https://github.com/ClickHouse/ClickHouse/issues/29677). [#29882](https://github.com/ClickHouse/ClickHouse/pull/29882) ([kevin wan](https://github.com/MaxWk)). +* Added columns `data_compressed_bytes`, `data_uncompressed_bytes`, `marks_bytes` into `system.data_skipping_indices`. Added columns `secondary_indices_compressed_bytes`, `secondary_indices_uncompressed_bytes`, `secondary_indices_marks_bytes` into `system.parts`. Closes [#29697](https://github.com/ClickHouse/ClickHouse/issues/29697). [#29896](https://github.com/ClickHouse/ClickHouse/pull/29896) ([Maksim Kita](https://github.com/kitaisreal)). +* User can now create dictionaries with comments: `CREATE DICTIONARY ... COMMENT 'vaue'` ... [#29899](https://github.com/ClickHouse/ClickHouse/pull/29899) ([Vasily Nemkov](https://github.com/Enmk)). +* Add support for FreeBSD aarch64. [#29952](https://github.com/ClickHouse/ClickHouse/pull/29952) ([MikaelUrankar](https://github.com/MikaelUrankar)). +* Added function `tokens`. That allow to split string into tokens using non-alpha numeric ASCII characters as separators. [#29981](https://github.com/ClickHouse/ClickHouse/pull/29981) ([Maksim Kita](https://github.com/kitaisreal)). +* Added new JSONExtractKeys function ... [#30056](https://github.com/ClickHouse/ClickHouse/pull/30056) ([Vitaly Orlov](https://github.com/orloffv)). +* Added `HashedArray` dictionary type. Closes [#30236](https://github.com/ClickHouse/ClickHouse/issues/30236). [#30242](https://github.com/ClickHouse/ClickHouse/pull/30242) ([Maksim Kita](https://github.com/kitaisreal)). +* Add ability to change nodes configuration (in `.xml` file) for ClickHouse Keeper. [#30372](https://github.com/ClickHouse/ClickHouse/pull/30372) ([alesapin](https://github.com/alesapin)). +* CompiledExpressionCache limit elements size using `compiled_expression_cache_elements_size` setting. [#30667](https://github.com/ClickHouse/ClickHouse/pull/30667) ([Maksim Kita](https://github.com/kitaisreal)). +* New function `mapExtractKeyLike` to get the map only kept elements matched specified pattern. [#30793](https://github.com/ClickHouse/ClickHouse/pull/30793) ([凌涛](https://github.com/lingtaolf)). +* Support `ALTER TABLE` for tables in `Memory` databases. [#30866](https://github.com/ClickHouse/ClickHouse/pull/30866) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### Performance Improvement +* * Remove branchy code in filter operation with a better implementation with popcnt/ctz which have better performance. [#29881](https://github.com/ClickHouse/ClickHouse/pull/29881) ([Jun Jin](https://github.com/vesslanjin)). +* - To take advantage of X86_64 feature, Use AVX2/AVX512 instructions to accelerate filter operation. [#30014](https://github.com/ClickHouse/ClickHouse/pull/30014) ([jasperzhu](https://github.com/jinjunzh)). +* Improve performance of aggregation in order of primary key (with enabled setting `optimize_aggregation_in_order`). [#30266](https://github.com/ClickHouse/ClickHouse/pull/30266) ([Anton Popov](https://github.com/CurtizJ)). +* ColumnDecimal improve performance for filter operation. [#30431](https://github.com/ClickHouse/ClickHouse/pull/30431) ([Jun Jin](https://github.com/vesslanjin)). +* Dictionaries support read from multiple threads. [#30500](https://github.com/ClickHouse/ClickHouse/pull/30500) ([Maksim Kita](https://github.com/kitaisreal)). +* - Improve filter bitmask generator function all in one with sse/avx2/avx512 instructions. [#30670](https://github.com/ClickHouse/ClickHouse/pull/30670) ([jasperzhu](https://github.com/jinjunzh)). +* Queries with `INTO OUTFILE` in `clickhouse-client` will use multiple threads. Fix the issue with flickering progress-bar when using `INTO OUTFILE`. This closes [#30873](https://github.com/ClickHouse/ClickHouse/issues/30873). This closes [#30872](https://github.com/ClickHouse/ClickHouse/issues/30872). [#30886](https://github.com/ClickHouse/ClickHouse/pull/30886) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Allow to write number in query as binary literal. Example `SELECT 0b001;`. [#29304](https://github.com/ClickHouse/ClickHouse/pull/29304) ([Maksim Kita](https://github.com/kitaisreal)). +* Pass through initial query_id for clickhouse-benchmark (previously if you run remote query via `clickhouse-benchmark`, queries on shards will not be linked to the initial query via `initial_query_id`). [#29364](https://github.com/ClickHouse/ClickHouse/pull/29364) ([Azat Khuzhin](https://github.com/azat)). +* fix lost in memory part when freeze. [#29376](https://github.com/ClickHouse/ClickHouse/pull/29376) ([Mo Xuan](https://github.com/mo-avatar)). +* Added setting `use_skip_indexes`. [#29405](https://github.com/ClickHouse/ClickHouse/pull/29405) ([Maksim Kita](https://github.com/kitaisreal)). +* Apply config changes to `max_concurrent_queries` during runtime (no need to restart). [#29414](https://github.com/ClickHouse/ClickHouse/pull/29414) ([Raúl Marín](https://github.com/Algunenano)). +* Transform `isNull`/`isNotNull` to `IS NULL`/`IS NOT NULL` (for external dbs, i.e. MySQL). [#29446](https://github.com/ClickHouse/ClickHouse/pull/29446) ([Azat Khuzhin](https://github.com/azat)). +* Enable per-query memory profiler (set to memory_profiler_step=4MiB) globally. [#29455](https://github.com/ClickHouse/ClickHouse/pull/29455) ([Azat Khuzhin](https://github.com/azat)). +* Add support for `IS NULL`/`IS NOT NULL` for external dbs (i.e. MySQL). [#29463](https://github.com/ClickHouse/ClickHouse/pull/29463) ([Azat Khuzhin](https://github.com/azat)). +* Fixed `There is no query or query context has expired` error in mutations with nested subqueries. Do not allow subqueries in mutation if table is replicated and `allow_nondeterministic_mutations` setting is disabled. [#29495](https://github.com/ClickHouse/ClickHouse/pull/29495) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Get rid of pointless restriction on projection name. Now projection name can start with `tmp_`. [#29520](https://github.com/ClickHouse/ClickHouse/pull/29520) ([Amos Bird](https://github.com/amosbird)). +* Reload dictionaries, models, user defined executable functions if servers config `dictionaries_config`, `models_config`, `user_defined_executable_functions_config` changes. Closes [#28142](https://github.com/ClickHouse/ClickHouse/issues/28142). [#29529](https://github.com/ClickHouse/ClickHouse/pull/29529) ([Maksim Kita](https://github.com/kitaisreal)). +* Allow user to change log levels without restart. [#29586](https://github.com/ClickHouse/ClickHouse/pull/29586) ([Nikolay Degterinsky](https://github.com/evillique)). +* Increase `listen_backlog` by default (to match default in newer linux kernel). [#29643](https://github.com/ClickHouse/ClickHouse/pull/29643) ([Azat Khuzhin](https://github.com/azat)). +* Fix incorrect nullable processing of JSONFunctions. This fixes [#29615](https://github.com/ClickHouse/ClickHouse/issues/29615) . Mark as improvement because https://github.com/ClickHouse/ClickHouse/pull/28012 is not released. [#29659](https://github.com/ClickHouse/ClickHouse/pull/29659) ([Amos Bird](https://github.com/amosbird)). +* Fixed the issue: `clickhouse-format --obfuscate` cannot process queries with embedded dictionaries (functions `regionTo...`). [#29667](https://github.com/ClickHouse/ClickHouse/pull/29667) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Query obfuscator `clickhouse-format --obfuscate` now works with more types of queries. [#29672](https://github.com/ClickHouse/ClickHouse/pull/29672) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `url` table function to process multiple URLs in parallel. This closes [#29670](https://github.com/ClickHouse/ClickHouse/issues/29670) and closes [#29671](https://github.com/ClickHouse/ClickHouse/issues/29671). [#29673](https://github.com/ClickHouse/ClickHouse/pull/29673) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add ClickHouse logo to Play UI. [#29674](https://github.com/ClickHouse/ClickHouse/pull/29674) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Apply `max_untracked_memory`/`memory_profiler_step`/`memory_profiler_sample_probability` during mutate/merge. [#29681](https://github.com/ClickHouse/ClickHouse/pull/29681) ([Azat Khuzhin](https://github.com/azat)). +* Remove some redundant `seek` calls while reading compressed blocks in MergeTree table engines family. [#29766](https://github.com/ClickHouse/ClickHouse/pull/29766) ([alesapin](https://github.com/alesapin)). +* Web UI: render bars in table cells. [#29792](https://github.com/ClickHouse/ClickHouse/pull/29792) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added partitioned table prefix 'p' for the query for fetching replica identity index. [#29828](https://github.com/ClickHouse/ClickHouse/pull/29828) ([Shoh Jahon](https://github.com/Shohjahon)). +* Add ability to trace peak memory usage (with new trace_type - MemoryPeak). [#29858](https://github.com/ClickHouse/ClickHouse/pull/29858) ([Azat Khuzhin](https://github.com/azat)). +* Remove unused variable in s3cluster. [#29912](https://github.com/ClickHouse/ClickHouse/pull/29912) ([李扬](https://github.com/taiyang-li)). +* Add `shutdown_wait_unfinished_queries` server setting to allowing waiting for running queries up to `shutdown_wait_unfinished` time. This is for [#24451](https://github.com/ClickHouse/ClickHouse/issues/24451) . [#29914](https://github.com/ClickHouse/ClickHouse/pull/29914) ([Amos Bird](https://github.com/amosbird)). +* Now min-max aggregation over the first expression of primary key is optimized by projection. This is for https://github.com/ClickHouse/ClickHouse/issues/329. [#29918](https://github.com/ClickHouse/ClickHouse/pull/29918) ([Amos Bird](https://github.com/amosbird)). +* Add ability to configure retries and delays between them for `clickhouse-copier`. [#29921](https://github.com/ClickHouse/ClickHouse/pull/29921) ([Azat Khuzhin](https://github.com/azat)). +* Update zoneinfo files to 2021c. [#29925](https://github.com/ClickHouse/ClickHouse/pull/29925) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better exception message while reading column from Arrow-supported formats like `Arrow`, `ArrowStream`, `Parquet` and `ORC`. This closes [#29926](https://github.com/ClickHouse/ClickHouse/issues/29926). [#29927](https://github.com/ClickHouse/ClickHouse/pull/29927) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix lock-order-inversion between periodic dictionary reload and config reload. [#29928](https://github.com/ClickHouse/ClickHouse/pull/29928) ([Azat Khuzhin](https://github.com/azat)). +* Fix `lock-order-inversion` between `DROP TABLE` for `DatabaseMemory` and `LiveView`. [#29929](https://github.com/ClickHouse/ClickHouse/pull/29929) ([Azat Khuzhin](https://github.com/azat)). +* Fix data-race between flush() and startup() in StorageBuffer. [#29930](https://github.com/ClickHouse/ClickHouse/pull/29930) ([Azat Khuzhin](https://github.com/azat)). +* Now clickhouse use DNS cache while communicating with external S3. [#29999](https://github.com/ClickHouse/ClickHouse/pull/29999) ([alesapin](https://github.com/alesapin)). +* ProfileEvents::Counters snapshot doesn't store data in std::atomic anymore. [#30000](https://github.com/ClickHouse/ClickHouse/pull/30000) ([Dmitry Novik](https://github.com/novikd)). +* Add ability to print raw profile events to `clickhouse-client` (This can be useful for debugging and for testing). [#30064](https://github.com/ClickHouse/ClickHouse/pull/30064) ([Azat Khuzhin](https://github.com/azat)). +* Improve solution https://github.com/ClickHouse/ClickHouse/pull/28853 See also https://github.com/ClickHouse/ClickHouse/pull/29928. [#30084](https://github.com/ClickHouse/ClickHouse/pull/30084) ([Vitaly Baranov](https://github.com/vitlibar)). +* Reduce amount of redundant compressed data read from disk for some types `SELECT` queries (only for MergeTree engines family). [#30111](https://github.com/ClickHouse/ClickHouse/pull/30111) ([alesapin](https://github.com/alesapin)). +* PolygonDictionary added support for read method if setting `store_polygon_key_column` = true. Closes [#30090](https://github.com/ClickHouse/ClickHouse/issues/30090). [#30142](https://github.com/ClickHouse/ClickHouse/pull/30142) ([Maksim Kita](https://github.com/kitaisreal)). +* Now clickhouse-client supports native multi-line editing. [#30143](https://github.com/ClickHouse/ClickHouse/pull/30143) ([Amos Bird](https://github.com/amosbird)). +* Now `Keeper` (as part of `clickhouse-server`) will start asynchronously if it can connect to some other node. [#30170](https://github.com/ClickHouse/ClickHouse/pull/30170) ([alesapin](https://github.com/alesapin)). +* Support SQL user defined functions for clickhouse-local. [#30179](https://github.com/ClickHouse/ClickHouse/pull/30179) ([Maksim Kita](https://github.com/kitaisreal)). +* Allow to remove `SAMPLE BY` expression from `MergeTree` tables (`ALTER TABLE REMOVE SAMPLE BY`). [#30180](https://github.com/ClickHouse/ClickHouse/pull/30180) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed comparison of Date32 with Date, DateTime, DateTime64, String. [#30219](https://github.com/ClickHouse/ClickHouse/pull/30219) ([liang.huang](https://github.com/lhuang09287750)). +* Allow symlinks to files in user_files directory for file table function. [#30309](https://github.com/ClickHouse/ClickHouse/pull/30309) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor improvements in replica cloning and enqueuing fetch for broken parts, that should avoid extremely rare hanging of `GET_PART` entries in replication queue. [#30346](https://github.com/ClickHouse/ClickHouse/pull/30346) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Added an ability to use FINAL clause in SELECT queries from GraphiteMergeTree. [#30360](https://github.com/ClickHouse/ClickHouse/pull/30360) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Allow extract non-string element as string using JSONExtractString. This is for https://github.com/ClickHouse/ClickHouse/pull/25452#issuecomment-927123287. [#30426](https://github.com/ClickHouse/ClickHouse/pull/30426) ([Amos Bird](https://github.com/amosbird)). +* More full support of positional arguments. [#30433](https://github.com/ClickHouse/ClickHouse/pull/30433) ([Kseniia Sumarokova](https://github.com/kssenii)). +* SQLUserDefinedFunctions support lambdas. Example `CREATE FUNCTION lambda_function AS x -> arrayMap(element -> element * 2, x);`. [#30435](https://github.com/ClickHouse/ClickHouse/pull/30435) ([Maksim Kita](https://github.com/kitaisreal)). +* SQLUserDefinedFunctions added DROP IF EXISTS support. Example `DROP FUNCTION IF EXISTS test_function`. [#30437](https://github.com/ClickHouse/ClickHouse/pull/30437) ([Maksim Kita](https://github.com/kitaisreal)). +* SQLUserDefinedFunctions support `CREATE OR REPLACE`, `CREATE IF NOT EXISTS` syntaxes. [#30454](https://github.com/ClickHouse/ClickHouse/pull/30454) ([Maksim Kita](https://github.com/kitaisreal)). +* Make query, which fetched table structure for PostgreSQL database because, more reliable. [#30477](https://github.com/ClickHouse/ClickHouse/pull/30477) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Get memory amount with Docker/cgroups limitations. See [#25662](https://github.com/ClickHouse/ClickHouse/issues/25662). [#30574](https://github.com/ClickHouse/ClickHouse/pull/30574) ([Pavel Medvedev](https://github.com/pmed)). +* SQLUserDefinedFunctions support ON CLUSTER. Example `CREATE FUNCTION test_function ON CLUSTER 'cluster' AS x -> x + 1;`. Closes [#30666](https://github.com/ClickHouse/ClickHouse/issues/30666). [#30734](https://github.com/ClickHouse/ClickHouse/pull/30734) ([Maksim Kita](https://github.com/kitaisreal)). +* Arrays of all serializable types are now supported by arrayStringConcat. [#30840](https://github.com/ClickHouse/ClickHouse/pull/30840) ([Nikita Taranov](https://github.com/nickitat)). +* Allow to parse values of `Date` data type in text formats as `YYYYMMDD` in addition to `YYYY-MM-DD`. This closes [#30870](https://github.com/ClickHouse/ClickHouse/issues/30870). [#30871](https://github.com/ClickHouse/ClickHouse/pull/30871) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rename the columns of SessionLog: `session_id` -> `auth_id`, `session_name` -> `session_id`, `changed_settings` -> `settings`. [#30882](https://github.com/ClickHouse/ClickHouse/pull/30882) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Bug Fix +* Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix releasing query ID and session ID at the end of query processing while handing gRPC call. This PR fixes flaky test [test_grpc_protocol/test.py::test_session](https://clickhouse-test-reports.s3.yandex.net/0/1ac03811a2df9717fa7c633d1af03def821d24b6/integration_tests_(memory).html). [#29954](https://github.com/ClickHouse/ClickHouse/pull/29954) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix reading from empty file on encrypted disk. [#30494](https://github.com/ClickHouse/ClickHouse/pull/30494) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Build/Testing/Packaging Improvement +* N/A Detailed description / Documentation draft:. [#29367](https://github.com/ClickHouse/ClickHouse/pull/29367) ([kevin wan](https://github.com/MaxWk)). +* Add ability to compile using newer version fo glibc w/o using new symbols. [#29594](https://github.com/ClickHouse/ClickHouse/pull/29594) ([Azat Khuzhin](https://github.com/azat)). +* Turning on experimental constexpr expressions evaluator for clang to speed up template code compilation. [#29668](https://github.com/ClickHouse/ClickHouse/pull/29668) ([Mike Kot](https://github.com/myrrc)). +* Fix an build error because of [Rename "common" to "base"](https://github.com/ClickHouse/ClickHouse/commit/fe6b7c77c7d6bd2a45a20f3b6bb4eb91da6177ff). [#29688](https://github.com/ClickHouse/ClickHouse/pull/29688) ([Sergei Semin](https://github.com/syominsergey)). +* Leave only required files in cross-compile toolchains. Include them as submodules (earlier they were downloaded as tarballs). [#29974](https://github.com/ClickHouse/ClickHouse/pull/29974) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add cross-build for PowerPC (powerpc64le). This closes [#9589](https://github.com/ClickHouse/ClickHouse/issues/9589). Enable support for interaction with MySQL for AArch64 and PowerPC. This closes [#26301](https://github.com/ClickHouse/ClickHouse/issues/26301). [#30010](https://github.com/ClickHouse/ClickHouse/pull/30010) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable Protobuf, Arrow, ORC, Parquet for AArch64 and Darwin builds. This closes [#29248](https://github.com/ClickHouse/ClickHouse/issues/29248). This closes [#28018](https://github.com/ClickHouse/ClickHouse/issues/28018). [#30015](https://github.com/ClickHouse/ClickHouse/pull/30015) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add tests for encrypted disk & encryption codec and ReplicatedMergeTree. [#30172](https://github.com/ClickHouse/ClickHouse/pull/30172) ([Vitaly Baranov](https://github.com/vitlibar)). +* ClickHouse can be statically built with Musl. This is added as experiment, it does not support building `odbc-bridge`, `library-bridge`, integration with CatBoost and some libraries. [#30248](https://github.com/ClickHouse/ClickHouse/pull/30248) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Recursive submodules are no longer needed for ClickHouse. [#30315](https://github.com/ClickHouse/ClickHouse/pull/30315) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added docker image to build docs. [#30499](https://github.com/ClickHouse/ClickHouse/pull/30499) ([Ilya Yatsishin](https://github.com/qoega)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release + +* Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Fix hanging DDL queries on Replicated database while adding a new replica. [#29328](https://github.com/ClickHouse/ClickHouse/pull/29328) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Fix vertical merges of projection parts. This fixes [#29253](https://github.com/ClickHouse/ClickHouse/issues/29253) . This PR also fixes several projection merge/mutation issues introduced in https://github.com/ClickHouse/ClickHouse/pull/25165. [#29337](https://github.com/ClickHouse/ClickHouse/pull/29337) ([Amos Bird](https://github.com/amosbird)). +* Remove window function `nth_value` as it is not memory-safe. This closes [#29347](https://github.com/ClickHouse/ClickHouse/issues/29347). [#29348](https://github.com/ClickHouse/ClickHouse/pull/29348) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix replicated access storage not shutting down cleanly when misconfigured. [#29388](https://github.com/ClickHouse/ClickHouse/pull/29388) ([Kevin Michel](https://github.com/kmichel-aiven)). +* rocksdb: fix race condition during multiple DB opening (and get back some tests that triggers the problem on CI). [#29393](https://github.com/ClickHouse/ClickHouse/pull/29393) ([Azat Khuzhin](https://github.com/azat)). +* Fix Logical error `Cannot capture columns` in functions greatest/least. Closes [#29334](https://github.com/ClickHouse/ClickHouse/issues/29334). [#29454](https://github.com/ClickHouse/ClickHouse/pull/29454) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix possible `Block structure mismatch` for subqueries with pushed-down `HAVING` predicate. Fixes [#29010](https://github.com/ClickHouse/ClickHouse/issues/29010). [#29475](https://github.com/ClickHouse/ClickHouse/pull/29475) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed incorrect table name parsing on loading of `Lazy` database. Fixes [#29456](https://github.com/ClickHouse/ClickHouse/issues/29456). [#29476](https://github.com/ClickHouse/ClickHouse/pull/29476) ([Alexander Tokmakov](https://github.com/tavplubix)). +* In ODBC bridge add retries for error Invalid cursor state. It is a retriable error. Closes [#29473](https://github.com/ClickHouse/ClickHouse/issues/29473). [#29518](https://github.com/ClickHouse/ClickHouse/pull/29518) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix bug in check `pathStartsWith` becuase there was bug with the usage of `std::mismatch`: ` The behavior is undefined if the second range is shorter than the first range.`. [#29531](https://github.com/ClickHouse/ClickHouse/pull/29531) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#29544](https://github.com/ClickHouse/ClickHouse/pull/29544) ([Raúl Marín](https://github.com/Algunenano)). +* Fix null deference for `GROUP BY WITH TOTALS HAVING` (when the column from `HAVING` wasn't selected). [#29553](https://github.com/ClickHouse/ClickHouse/pull/29553) ([Azat Khuzhin](https://github.com/azat)). +* Fix rare segfault in `ALTER MODIFY` query when using incorrect table identifier in `DEFAULT` expression like `x.y.z...` Fixes [#29184](https://github.com/ClickHouse/ClickHouse/issues/29184). [#29573](https://github.com/ClickHouse/ClickHouse/pull/29573) ([alesapin](https://github.com/alesapin)). +* Fix JIT expression compilation with aliases and short-circuit expression evaluation. Closes [#29403](https://github.com/ClickHouse/ClickHouse/issues/29403). [#29574](https://github.com/ClickHouse/ClickHouse/pull/29574) ([Maksim Kita](https://github.com/kitaisreal)). +* Condition in filter predicate could be lost after push-down optimisation. [#29625](https://github.com/ClickHouse/ClickHouse/pull/29625) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed incorrect behaviour of setting `materialized_postgresql_tables_list` at server restart. Found in [#28529](https://github.com/ClickHouse/ClickHouse/issues/28529). [#29686](https://github.com/ClickHouse/ClickHouse/pull/29686) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix incorrect `GROUP BY` (multiple rows with the same keys in result) in case of distributed query when shards had mixed versions `<= 21.3` and `>= 21.4`, `GROUP BY` key had several columns all with fixed size, and two-level aggregation was activated (see `group_by_two_level_threshold` and `group_by_two_level_threshold_bytes`). Fixes [#29580](https://github.com/ClickHouse/ClickHouse/issues/29580). [#29735](https://github.com/ClickHouse/ClickHouse/pull/29735) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix concurrent access to `LowCardinality` during `GROUP BY` (leads to SIGSEGV). [#29782](https://github.com/ClickHouse/ClickHouse/pull/29782) ([Azat Khuzhin](https://github.com/azat)). +* Fix bad cast in `ATTACH TABLE ... FROM 'path'` query when non-string literal is used instead of path. It may lead to reading of uninitialized memory. [#29790](https://github.com/ClickHouse/ClickHouse/pull/29790) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid `Timeout exceeded: elapsed 18446744073.709553 seconds` error that might happen in extremely rare cases, presumably due to some bug in kernel. Fixes [#29154](https://github.com/ClickHouse/ClickHouse/issues/29154). [#29811](https://github.com/ClickHouse/ClickHouse/pull/29811) ([Alexander Tokmakov](https://github.com/tavplubix)). +* MaterializedMySQL: Fix an issue where if the connection to MySQL was lost, only parts of a transaction could be processed. [#29837](https://github.com/ClickHouse/ClickHouse/pull/29837) ([Håvard Kvålen](https://github.com/havardk)). +* Fix system tables recreation check (fails to detect changes in enum values). [#29857](https://github.com/ClickHouse/ClickHouse/pull/29857) ([Azat Khuzhin](https://github.com/azat)). +* Fix potential resource leak of the concurrent query limit of merge tree tables introduced in https://github.com/ClickHouse/ClickHouse/pull/19544 . [#29879](https://github.com/ClickHouse/ClickHouse/pull/29879) ([Amos Bird](https://github.com/amosbird)). +* Fix data-race between `LogSink::writeMarks()` and `LogSource` in `StorageLog`. [#29946](https://github.com/ClickHouse/ClickHouse/pull/29946) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible data-race between `FileChecker` and `StorageLog`/`StorageStripeLog`. [#29959](https://github.com/ClickHouse/ClickHouse/pull/29959) ([Azat Khuzhin](https://github.com/azat)). +* try to close issue: [#29965](https://github.com/ClickHouse/ClickHouse/issues/29965). [#29976](https://github.com/ClickHouse/ClickHouse/pull/29976) ([hexiaoting](https://github.com/hexiaoting)). +* Fix crash of sample by `tuple()`, closes [#30004](https://github.com/ClickHouse/ClickHouse/issues/30004). [#30016](https://github.com/ClickHouse/ClickHouse/pull/30016) ([flynn](https://github.com/ucasfl)). +* Dropped `Memory` database might reappear after server restart, it's fixed ([#29795](https://github.com/ClickHouse/ClickHouse/issues/29795)). Also added `force_remove_data_recursively_on_drop` setting as a workaround for `Directory not empty` error when dropping `Ordinary` database (because it's not possible to remove data leftovers manually in cloud environment). [#30054](https://github.com/ClickHouse/ClickHouse/pull/30054) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix race between MOVE PARTITION and merges/mutations for MergeTree. [#30074](https://github.com/ClickHouse/ClickHouse/pull/30074) ([Azat Khuzhin](https://github.com/azat)). +* Fix error `Port is already connected` for queries with `GLOBAL IN` and `WITH TOTALS`. Only for 21.9 and 21.10. [#30086](https://github.com/ClickHouse/ClickHouse/pull/30086) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Support nullable arguments in function `initializeAggregation`. [#30177](https://github.com/ClickHouse/ClickHouse/pull/30177) ([Anton Popov](https://github.com/CurtizJ)). +* Fix INSERT SELECT incorrectly fills MATERIALIZED column based of Nullable column. [#30189](https://github.com/ClickHouse/ClickHouse/pull/30189) ([Azat Khuzhin](https://github.com/azat)). +* Fix `pread_fake_async`/`pread_threadpool` with `min_bytes_to_use_direct_io`. [#30191](https://github.com/ClickHouse/ClickHouse/pull/30191) ([Azat Khuzhin](https://github.com/azat)). +* Fix reading from `MergeTree` with `max_read_buffer_size=0` (can lead to `Can't adjust last granule` `LOGICAL_ERROR`, or even data loss). [#30192](https://github.com/ClickHouse/ClickHouse/pull/30192) ([Azat Khuzhin](https://github.com/azat)). +* * Allow identifiers staring with numbers in multiple joins. [#30230](https://github.com/ClickHouse/ClickHouse/pull/30230) ([Vladimir C](https://github.com/vdimir)). +* FlatDictionary, HashedDictionary fix bytes_allocated calculation for nullable attributes. [#30238](https://github.com/ClickHouse/ClickHouse/pull/30238) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix crash with shortcircuit and lowcardinality in multiIf. [#30243](https://github.com/ClickHouse/ClickHouse/pull/30243) ([Raúl Marín](https://github.com/Algunenano)). +* Fix `[I]LIKE` function. Closes [#28661](https://github.com/ClickHouse/ClickHouse/issues/28661). [#30244](https://github.com/ClickHouse/ClickHouse/pull/30244) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix ComplexKeyHashedDictionary, ComplexKeySparseHashedDictionary parsing `preallocate` option from layout config. [#30246](https://github.com/ClickHouse/ClickHouse/pull/30246) ([Maksim Kita](https://github.com/kitaisreal)). +* fix replaceRegexpAll bug. [#30292](https://github.com/ClickHouse/ClickHouse/pull/30292) ([Memo](https://github.com/Joeywzr)). +* Fix column alias resolution of JOIN queries when projection is enabled. This fixes [#30146](https://github.com/ClickHouse/ClickHouse/issues/30146). [#30293](https://github.com/ClickHouse/ClickHouse/pull/30293) ([Amos Bird](https://github.com/amosbird)). +* Queries with condition like `IN (subquery)` could return incorrect result in case if aggregate projection applied. Fixed creation of sets for projections. [#30310](https://github.com/ClickHouse/ClickHouse/pull/30310) ([Amos Bird](https://github.com/amosbird)). +* Makes [#30162](https://github.com/ClickHouse/ClickHouse/issues/30162) less possible ... [#30370](https://github.com/ClickHouse/ClickHouse/pull/30370) ([Vasily Nemkov](https://github.com/Enmk)). +* Fixed segfault which might happen if session expired during execution of REPLACE PARTITION. [#30432](https://github.com/ClickHouse/ClickHouse/pull/30432) ([Alexander Tokmakov](https://github.com/tavplubix)). +* * Fix deadlock on ALTER with scalar subquery to the same table, close [#30461](https://github.com/ClickHouse/ClickHouse/issues/30461). [#30492](https://github.com/ClickHouse/ClickHouse/pull/30492) ([Vladimir C](https://github.com/vdimir)). +* Add missing parenthesis for `isNotNull`/`isNull` rewrites to `IS [NOT] NULL` (fixes queries that has something like `isNotNull(1)+isNotNull(2)`). [#30520](https://github.com/ClickHouse/ClickHouse/pull/30520) ([Azat Khuzhin](https://github.com/azat)). +* Limit push down optimization could cause a error `Cannot find column`. Fixes [#30438](https://github.com/ClickHouse/ClickHouse/issues/30438). [#30562](https://github.com/ClickHouse/ClickHouse/pull/30562) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update aws-sdk submodule for throttling in Yandex.S3. [#30646](https://github.com/ClickHouse/ClickHouse/pull/30646) ([ianton-ru](https://github.com/ianton-ru)). +* Functions for case-insensitive search in UTF8 strings like `positionCaseInsensitiveUTF8` and `countSubstringsCaseInsensitiveUTF8` might find substrings that actually does not match, it's fixed. [#30663](https://github.com/ClickHouse/ClickHouse/pull/30663) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix PREWHERE with WHERE in case of always true PREWHERE. [#30668](https://github.com/ClickHouse/ClickHouse/pull/30668) ([Azat Khuzhin](https://github.com/azat)). +* Fixed a race condition between `REPLACE/MOVE PARTITION` and background merge in non-replicated `MergeTree` that might cause a part of moved/replaced data to remain in partition. Fixes [#29327](https://github.com/ClickHouse/ClickHouse/issues/29327). [#30717](https://github.com/ClickHouse/ClickHouse/pull/30717) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Clean temporary directory when localBackup failed by some reason. [#30797](https://github.com/ClickHouse/ClickHouse/pull/30797) ([ianton-ru](https://github.com/ianton-ru)). +* Fixed ambiguity when extracting auxiliary ZooKeeper name from ZooKeeper path in `ReplicatedMergeTree`. Previously server might fail to start with `Unknown auxiliary ZooKeeper name` if ZooKeeper path contains a colon. Fixes [#29052](https://github.com/ClickHouse/ClickHouse/issues/29052). Also it was allowed to specify ZooKeeper path that does not start with slash, but now it's deprecated and creation of new tables with such path is not allowed. Slashes and colons in auxiliary ZooKeeper names are not allowed too. [#30822](https://github.com/ClickHouse/ClickHouse/pull/30822) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix crash when projection with hashing function is materialized. This fixes [#30861](https://github.com/ClickHouse/ClickHouse/issues/30861) . The issue is similar to https://github.com/ClickHouse/ClickHouse/pull/28560 which is a lack of proper understanding of the invariant of header's emptyness. [#30877](https://github.com/ClickHouse/ClickHouse/pull/30877) ([Amos Bird](https://github.com/amosbird)). +* Fix set index not used in AND/OR expressions when there are more than two operands. This fixes [#30416](https://github.com/ClickHouse/ClickHouse/issues/30416) . [#30887](https://github.com/ClickHouse/ClickHouse/pull/30887) ([Amos Bird](https://github.com/amosbird)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Generate test_results.tsv for fasttest'. [#29319](https://github.com/ClickHouse/ClickHouse/pull/29319) ([Dmitry Novik](https://github.com/novikd)). +* NO CL ENTRY: 'Revert "Add coroutines example."'. [#29829](https://github.com/ClickHouse/ClickHouse/pull/29829) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* NO CL ENTRY: 'Link FAQ from Debian installation'. [#29836](https://github.com/ClickHouse/ClickHouse/pull/29836) ([Federico Ceratto](https://github.com/FedericoCeratto)). +* NO CL ENTRY: 'add support of window function in antlr grammar'. [#30181](https://github.com/ClickHouse/ClickHouse/pull/30181) ([PHaroZ](https://github.com/PHaroZ)). +* NO CL ENTRY: 'Revert "Fix style regressions on benchmark page"'. [#30652](https://github.com/ClickHouse/ClickHouse/pull/30652) ([alesapin](https://github.com/alesapin)). +* NO CL ENTRY: 'Revert "Improve usability of `remote_url_allow_hosts`"'. [#30707](https://github.com/ClickHouse/ClickHouse/pull/30707) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* NO CL ENTRY: 'Revert "Revert "Improve usability of `remote_url_allow_hosts`""'. [#30708](https://github.com/ClickHouse/ClickHouse/pull/30708) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* NO CL ENTRY: 'remove some unneeded header files'. [#30722](https://github.com/ClickHouse/ClickHouse/pull/30722) ([flynn](https://github.com/ucasfl)). + +#### Testing Improvement + +* Implemented structure-aware fuzzing approach in ClickHouse for select statement parser. [#30012](https://github.com/ClickHouse/ClickHouse/pull/30012) ([Paul](https://github.com/PaulCher)). + diff --git a/docs/changelogs/v21.11.10.1-stable.md b/docs/changelogs/v21.11.10.1-stable.md new file mode 100644 index 00000000000..f07eccd30c5 --- /dev/null +++ b/docs/changelogs/v21.11.10.1-stable.md @@ -0,0 +1,2 @@ +### ClickHouse release v21.11.10.1-stable FIXME as compared to v21.11.9.1-stable + diff --git a/docs/changelogs/v21.11.11.1-stable.md b/docs/changelogs/v21.11.11.1-stable.md new file mode 100644 index 00000000000..e46f43c53e0 --- /dev/null +++ b/docs/changelogs/v21.11.11.1-stable.md @@ -0,0 +1,6 @@ +### ClickHouse release v21.11.11.1-stable FIXME as compared to v21.11.10.1-stable + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#33656](https://github.com/ClickHouse/ClickHouse/issues/33656): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.11.2.2-stable.md b/docs/changelogs/v21.11.2.2-stable.md new file mode 100644 index 00000000000..f48f91a9b13 --- /dev/null +++ b/docs/changelogs/v21.11.2.2-stable.md @@ -0,0 +1,8 @@ +### ClickHouse release v21.11.2.2-stable FIXME as compared to v21.11.1.8636-prestable + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#31154](https://github.com/ClickHouse/ClickHouse/issues/31154): Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)). +* Backported in [#31027](https://github.com/ClickHouse/ClickHouse/issues/31027): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31026](https://github.com/ClickHouse/ClickHouse/issues/31026): Fix JSONValue/Query with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.11.3.6-stable.md b/docs/changelogs/v21.11.3.6-stable.md new file mode 100644 index 00000000000..bf46ecec590 --- /dev/null +++ b/docs/changelogs/v21.11.3.6-stable.md @@ -0,0 +1,10 @@ +### ClickHouse release v21.11.3.6-stable FIXME as compared to v21.11.2.2-stable + +#### Bug Fix +* Backported in [#31246](https://github.com/ClickHouse/ClickHouse/issues/31246): Memory amount was incorrectly estimated when ClickHouse is run in containers with cgroup limits. [#31157](https://github.com/ClickHouse/ClickHouse/pull/31157) ([Pavel Medvedev](https://github.com/pmed)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#31206](https://github.com/ClickHouse/ClickHouse/issues/31206): Fix possible assert in `hdfs` table function/engine, add test. [#31036](https://github.com/ClickHouse/ClickHouse/pull/31036) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31202](https://github.com/ClickHouse/ClickHouse/issues/31202): Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.11.4.14-stable.md b/docs/changelogs/v21.11.4.14-stable.md new file mode 100644 index 00000000000..b3d44b8c193 --- /dev/null +++ b/docs/changelogs/v21.11.4.14-stable.md @@ -0,0 +1,17 @@ +### ClickHouse release v21.11.4.14-stable FIXME as compared to v21.11.3.6-stable + +#### Bug Fix +* Backported in [#31370](https://github.com/ClickHouse/ClickHouse/issues/31370): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#31282](https://github.com/ClickHouse/ClickHouse/issues/31282): Fix some corner cases with intersect/except. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31237](https://github.com/ClickHouse/ClickHouse/issues/31237): Fix bug which broke select queries if they happened after dropping materialized view. Found in [#30691](https://github.com/ClickHouse/ClickHouse/issues/30691). [#30997](https://github.com/ClickHouse/ClickHouse/pull/30997) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31374](https://github.com/ClickHouse/ClickHouse/issues/31374): Fix StorageMerge with aliases and where (it did not work before at all). Closes [#28802](https://github.com/ClickHouse/ClickHouse/issues/28802). [#31044](https://github.com/ClickHouse/ClickHouse/pull/31044) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31236](https://github.com/ClickHouse/ClickHouse/issues/31236): Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)). +* Backported in [#31435](https://github.com/ClickHouse/ClickHouse/issues/31435): Fix bug with group by and positional arguments. Closes [#31280](https://github.com/ClickHouse/ClickHouse/issues/31280)#issuecomment-968696186. [#31420](https://github.com/ClickHouse/ClickHouse/pull/31420) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Imrovement (changelog entry is not required) + +* Backported in [#31283](https://github.com/ClickHouse/ClickHouse/issues/31283): Rename setting value `read_threadpool` to `threadpool` for setting `remote_filesystem_read_method`. [#31224](https://github.com/ClickHouse/ClickHouse/pull/31224) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.11.5.33-stable.md b/docs/changelogs/v21.11.5.33-stable.md new file mode 100644 index 00000000000..973c443d9f8 --- /dev/null +++ b/docs/changelogs/v21.11.5.33-stable.md @@ -0,0 +1,33 @@ +### ClickHouse release v21.11.5.33-stable FIXME as compared to v21.11.4.14-stable + +#### Performance Improvement +* Backported in [#31735](https://github.com/ClickHouse/ClickHouse/issues/31735): Improve performance of JSON and XML output formats. [#31673](https://github.com/ClickHouse/ClickHouse/pull/31673) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#31572](https://github.com/ClickHouse/ClickHouse/issues/31572): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#31517](https://github.com/ClickHouse/ClickHouse/issues/31517): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)). +* Backported in [#31551](https://github.com/ClickHouse/ClickHouse/issues/31551): Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#31580](https://github.com/ClickHouse/ClickHouse/issues/31580): * Disable `partial_merge_join_left_table_buffer_bytes` before bug in this optimization is fixed. See [#31009](https://github.com/ClickHouse/ClickHouse/issues/31009)). * Remove redundant option `partial_merge_join_optimizations`. [#31528](https://github.com/ClickHouse/ClickHouse/pull/31528) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#31600](https://github.com/ClickHouse/ClickHouse/issues/31600): Fix invalid generated JSON when only column names contain invalid UTF-8 sequences. [#31534](https://github.com/ClickHouse/ClickHouse/pull/31534) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Backported in [#31593](https://github.com/ClickHouse/ClickHouse/issues/31593): All non-x86 builds were broken, because we don't have tests for them. This closes [#31417](https://github.com/ClickHouse/ClickHouse/issues/31417). This closes [#31524](https://github.com/ClickHouse/ClickHouse/issues/31524). [#31574](https://github.com/ClickHouse/ClickHouse/pull/31574) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#31795](https://github.com/ClickHouse/ClickHouse/issues/31795): Fix sparkbars are not aligned, see: [#26175](https://github.com/ClickHouse/ClickHouse/issues/26175)#issuecomment-960353867, [comment](https://github.com/ClickHouse/ClickHouse/issues/26175#issuecomment-961155065). [#31624](https://github.com/ClickHouse/ClickHouse/pull/31624) ([小路](https://github.com/nicelulu)). +* Backported in [#31747](https://github.com/ClickHouse/ClickHouse/issues/31747): `RENAME TABLE` query worked incorrectly on attempt to rename an DDL dictionary in `Ordinary` database, it's fixed. [#31638](https://github.com/ClickHouse/ClickHouse/pull/31638) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31791](https://github.com/ClickHouse/ClickHouse/issues/31791): Settings `input_format_allow_errors_num` and `input_format_allow_errors_ratio` did not work for parsing of domain types, such as `IPv4`, it's fixed. Fixes [#31686](https://github.com/ClickHouse/ClickHouse/issues/31686). [#31697](https://github.com/ClickHouse/ClickHouse/pull/31697) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31895](https://github.com/ClickHouse/ClickHouse/issues/31895): * Fixed function ngrams when string contains utf8 characters. [#31706](https://github.com/ClickHouse/ClickHouse/pull/31706) ([yandd](https://github.com/yandd)). +* Backported in [#31831](https://github.com/ClickHouse/ClickHouse/issues/31831): Fixed `there are no such cluster here` error on execution of `ON CLUSTER` query if specified cluster name is name of `Replicated` database. [#31723](https://github.com/ClickHouse/ClickHouse/pull/31723) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31814](https://github.com/ClickHouse/ClickHouse/issues/31814): Fix race in JSONEachRowWithProgress output format when data and lines with progress are mixed in output. [#31736](https://github.com/ClickHouse/ClickHouse/pull/31736) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#32015](https://github.com/ClickHouse/ClickHouse/issues/32015): Fixed rare segfault on concurrent `ATTACH PARTITION` queries. [#31738](https://github.com/ClickHouse/ClickHouse/pull/31738) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32017](https://github.com/ClickHouse/ClickHouse/issues/32017): Fix group by / order by / limit by aliases with positional arguments enabled. Closes [#31173](https://github.com/ClickHouse/ClickHouse/issues/31173). [#31741](https://github.com/ClickHouse/ClickHouse/pull/31741) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31760](https://github.com/ClickHouse/ClickHouse/issues/31760): Fix usage of `Buffer` table engine with type `Map`. Fixes [#30546](https://github.com/ClickHouse/ClickHouse/issues/30546). [#31742](https://github.com/ClickHouse/ClickHouse/pull/31742) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#31891](https://github.com/ClickHouse/ClickHouse/issues/31891): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#32029](https://github.com/ClickHouse/ClickHouse/issues/32029): Fix invalid cast of nullable type when nullable primary key is used. This fixes [#31075](https://github.com/ClickHouse/ClickHouse/issues/31075). [#31823](https://github.com/ClickHouse/ClickHouse/pull/31823) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#31920](https://github.com/ClickHouse/ClickHouse/issues/31920): Fix reading from `MergeTree` tables with enabled `use_uncompressed_cache`. [#31826](https://github.com/ClickHouse/ClickHouse/pull/31826) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#32076](https://github.com/ClickHouse/ClickHouse/issues/32076): Fix a bug about function transform with decimal args. [#31839](https://github.com/ClickHouse/ClickHouse/pull/31839) ([Shuai li](https://github.com/loneylee)). +* Backported in [#31938](https://github.com/ClickHouse/ClickHouse/issues/31938): - Change configuration path from `keeper_server.session_timeout_ms` to `keeper_server.coordination_settings.session_timeout_ms` when constructing a `KeeperTCPHandler` - Same with `operation_timeout`. [#31859](https://github.com/ClickHouse/ClickHouse/pull/31859) ([JackyWoo](https://github.com/JackyWoo)). +* Backported in [#31908](https://github.com/ClickHouse/ClickHouse/issues/31908): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#32091](https://github.com/ClickHouse/ClickHouse/issues/32091): Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32020](https://github.com/ClickHouse/ClickHouse/issues/32020): Fix FileLog engine unnesessary create meta data directory when create table failed. Fix [#31962](https://github.com/ClickHouse/ClickHouse/issues/31962). [#31967](https://github.com/ClickHouse/ClickHouse/pull/31967) ([flynn](https://github.com/ucasfl)). + diff --git a/docs/changelogs/v21.11.6.7-stable.md b/docs/changelogs/v21.11.6.7-stable.md new file mode 100644 index 00000000000..1f3df589466 --- /dev/null +++ b/docs/changelogs/v21.11.6.7-stable.md @@ -0,0 +1,24 @@ +### ClickHouse release v21.11.6.7-stable FIXME as compared to v21.11.5.33-stable + +#### Bug Fix +* Backported in [#32254](https://github.com/ClickHouse/ClickHouse/issues/32254): Fix skipping columns while writing protobuf. This PR fixes [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160), see the comment [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160)#issuecomment-980595318. [#31988](https://github.com/ClickHouse/ClickHouse/pull/31988) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#32345](https://github.com/ClickHouse/ClickHouse/issues/32345): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#32152](https://github.com/ClickHouse/ClickHouse/issues/32152): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#32298](https://github.com/ClickHouse/ClickHouse/issues/32298): Fix recursive user defined functions crash. Closes [#30856](https://github.com/ClickHouse/ClickHouse/issues/30856). [#31820](https://github.com/ClickHouse/ClickHouse/pull/31820) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#32148](https://github.com/ClickHouse/ClickHouse/issues/32148): Fixed `Directory ... already exists and is not empty` error when detaching part. [#32063](https://github.com/ClickHouse/ClickHouse/pull/32063) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32212](https://github.com/ClickHouse/ClickHouse/issues/32212): Fix `CAST` from `Nullable` with `cast_keep_nullable` (`PARAMETER_OUT_OF_BOUND` error before for i.e. `toUInt32OrDefault(toNullable(toUInt32(1)))`). [#32080](https://github.com/ClickHouse/ClickHouse/pull/32080) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#32283](https://github.com/ClickHouse/ClickHouse/issues/32283): Dictionaries fix cases when `{condition}` does not work for custom database queries. [#32117](https://github.com/ClickHouse/ClickHouse/pull/32117) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#32482](https://github.com/ClickHouse/ClickHouse/issues/32482): Fix 'APPLY lambda' parsing which could lead to client/server crash. [#32138](https://github.com/ClickHouse/ClickHouse/pull/32138) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#32217](https://github.com/ClickHouse/ClickHouse/issues/32217): Number of active replicas might be determined incorrectly when inserting with quorum if setting `replicated_can_become_leader` is disabled on some replicas. It's fixed. [#32157](https://github.com/ClickHouse/ClickHouse/pull/32157) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32311](https://github.com/ClickHouse/ClickHouse/issues/32311): XML dictionaries identifiers, used in table create query, can be qualified to `default_database` during upgrade to newer version. Closes [#31963](https://github.com/ClickHouse/ClickHouse/issues/31963). [#32187](https://github.com/ClickHouse/ClickHouse/pull/32187) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#32354](https://github.com/ClickHouse/ClickHouse/issues/32354): Fixed crash with SIGFPE in aggregate function `avgWeighted` with `Decimal` argument. Fixes [#32053](https://github.com/ClickHouse/ClickHouse/issues/32053). [#32303](https://github.com/ClickHouse/ClickHouse/pull/32303) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32392](https://github.com/ClickHouse/ClickHouse/issues/32392): Fix `ALTER ... MATERIALIZE COLUMN ...` queries in case when data type of default expression is not equal to the data type of column. [#32348](https://github.com/ClickHouse/ClickHouse/pull/32348) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#32418](https://github.com/ClickHouse/ClickHouse/issues/32418): Fixed the behavior when mutations that have nothing to do are stuck (with enabled setting `empty_result_for_aggregation_by_empty_set`). [#32358](https://github.com/ClickHouse/ClickHouse/pull/32358) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Manual backport of [#31766](https://github.com/ClickHouse/ClickHouse/issues/31766) into 21.11'. [#32202](https://github.com/ClickHouse/ClickHouse/pull/32202) ([Raúl Marín](https://github.com/Algunenano)). + diff --git a/docs/changelogs/v21.11.7.9-stable.md b/docs/changelogs/v21.11.7.9-stable.md new file mode 100644 index 00000000000..1d907ad0ce1 --- /dev/null +++ b/docs/changelogs/v21.11.7.9-stable.md @@ -0,0 +1,15 @@ +### ClickHouse release v21.11.7.9-stable FIXME as compared to v21.11.6.7-stable + +#### Bug Fix +* Backported in [#32691](https://github.com/ClickHouse/ClickHouse/issues/32691): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#32711](https://github.com/ClickHouse/ClickHouse/issues/32711): Fix failures in queries that are trying to use skipping indices, which are not materialized yet. Fixes [#32292](https://github.com/ClickHouse/ClickHouse/issues/32292) and [#30343](https://github.com/ClickHouse/ClickHouse/issues/30343). [#32359](https://github.com/ClickHouse/ClickHouse/pull/32359) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#32568](https://github.com/ClickHouse/ClickHouse/issues/32568): Fix crash in `JoinCommon::removeColumnNullability`, close [#32458](https://github.com/ClickHouse/ClickHouse/issues/32458). [#32508](https://github.com/ClickHouse/ClickHouse/pull/32508) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#32732](https://github.com/ClickHouse/ClickHouse/issues/32732): Fix surprisingly bad code in function `file`. [#32640](https://github.com/ClickHouse/ClickHouse/pull/32640) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release): + +* Backported in [#32617](https://github.com/ClickHouse/ClickHouse/issues/32617): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.8.4-stable.md b/docs/changelogs/v21.11.8.4-stable.md new file mode 100644 index 00000000000..0826b473758 --- /dev/null +++ b/docs/changelogs/v21.11.8.4-stable.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.11.8.4-stable FIXME as compared to v21.11.7.9-stable + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#32679](https://github.com/ClickHouse/ClickHouse/issues/32679): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#32543](https://github.com/ClickHouse/ClickHouse/issues/32543): Some replication queue entries might hang for `temporary_directories_lifetime` (1 day by default) with `Directory tmp_merge_` or `Part ... (state Deleting) already exists, but it will be deleted soon` or similar error. It's fixed. Fixes [#29616](https://github.com/ClickHouse/ClickHouse/issues/29616). [#32201](https://github.com/ClickHouse/ClickHouse/pull/32201) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32769](https://github.com/ClickHouse/ClickHouse/issues/32769): Fix sparse_hashed dict performance with sequential keys (wrong hash function). [#32536](https://github.com/ClickHouse/ClickHouse/pull/32536) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#32635](https://github.com/ClickHouse/ClickHouse/issues/32635): Fix table lifetime (i.e. possible use-after-free) in case of parallel DROP TABLE and INSERT. [#32572](https://github.com/ClickHouse/ClickHouse/pull/32572) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#32633](https://github.com/ClickHouse/ClickHouse/issues/32633): Fix possible exception at RabbitMQ storage startup by delaying channel creation. [#32584](https://github.com/ClickHouse/ClickHouse/pull/32584) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#32891](https://github.com/ClickHouse/ClickHouse/issues/32891): Fix LOGICAL_ERROR when the target of a materialized view is a JOIN or a SET table. [#32669](https://github.com/ClickHouse/ClickHouse/pull/32669) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#32792](https://github.com/ClickHouse/ClickHouse/issues/32792): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). + diff --git a/docs/changelogs/v21.11.9.1-stable.md b/docs/changelogs/v21.11.9.1-stable.md new file mode 100644 index 00000000000..c1754614c3c --- /dev/null +++ b/docs/changelogs/v21.11.9.1-stable.md @@ -0,0 +1,6 @@ +### ClickHouse release v21.11.9.1-stable FIXME as compared to v21.11.8.4-stable + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#33181](https://github.com/ClickHouse/ClickHouse/issues/33181): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.12.1.9017-prestable.md b/docs/changelogs/v21.12.1.9017-prestable.md new file mode 100644 index 00000000000..bcf5424fc63 --- /dev/null +++ b/docs/changelogs/v21.12.1.9017-prestable.md @@ -0,0 +1,206 @@ +### ClickHouse release v21.12.1.9017-prestable FIXME as compared to v21.11.1.8636-prestable + +#### Backward Incompatible Change +* Add custom null representation support for TSV/CSV input formats. Fix deserialing Nullable(String) in TSV/CSV/JSONCompactStringsEachRow/JSONStringsEachRow input formats. Rename `output_format_csv_null_representation` and `output_format_tsv_null_representation` to `format_csv_null_representation` and `format_tsv_null_representation` accordingly. [#30497](https://github.com/ClickHouse/ClickHouse/pull/30497) ([Kruglov Pavel](https://github.com/Avogar)). +* Return unquoted string in JSON_VALUE. Closes [#27965](https://github.com/ClickHouse/ClickHouse/issues/27965). [#31008](https://github.com/ClickHouse/ClickHouse/pull/31008) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Do not allow direct select for Kafka/RabbitMQ/FileLog. Can be enables by setting `stream_like_engine_allow_direct_select`. Direct select will be not allowed even if enabled by setting in case there is attached materialized view. For Kafka and RabbitMQ direct select if allowed, will not commit massages by default. To enable commits with direct select user must use storage level setting `kafka{rabbitmq}_commit_on_select=1` (default `0`). cc @filimonov. [#31053](https://github.com/ClickHouse/ClickHouse/pull/31053) ([Kseniia Sumarokova](https://github.com/kssenii)). +* A "leader election" mechanism is removed from `ReplicatedMergeTree`, because multiple leaders are supported since 20.6. If you are upgrading from older version and some replica with old version is a leader, then server will fail to start after upgrade. Stop replicas with old version to make new version start. After that it will not be possible to downgrade to version older than 20.6. [#32140](https://github.com/ClickHouse/ClickHouse/pull/32140) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### New Feature +* Support for Stream Processing. [#8331](https://github.com/ClickHouse/ClickHouse/pull/8331) ([vxider](https://github.com/Vxider)). +* - Added CONSTRAINT ... ASSUME ... (without checking during INSERT) - Added query transformation to CNF (https://github.com/ClickHouse/ClickHouse/issues/11749) for more convenient optimization - Added simple query rewriting using constraints (only simple matching now, will be improved to support <,=,>... later) - Added ability to replace heavy columns with light - Added ability to use the index in queries. [#18787](https://github.com/ClickHouse/ClickHouse/pull/18787) ([Nikita Vasilev](https://github.com/nikvas0)). +* * Add Map combinator for `Map` type. * Rename old `sum-, min-, max- Map` for mapped arrays to `sum-, min-, max- MappedArrays`. [#24539](https://github.com/ClickHouse/ClickHouse/pull/24539) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Support `JOIN ON 1 = 1` that have CROSS JOIN semantic, close [#25578](https://github.com/ClickHouse/ClickHouse/issues/25578). [#25894](https://github.com/ClickHouse/ClickHouse/pull/25894) ([Vladimir C](https://github.com/vdimir)). +* Adding function `getFuzzerData()` to easily fuzz particular functions. This closes [#23227](https://github.com/ClickHouse/ClickHouse/issues/23227). [#27526](https://github.com/ClickHouse/ClickHouse/pull/27526) ([Alexey Boykov](https://github.com/mathalex)). +* This closes [#28774](https://github.com/ClickHouse/ClickHouse/issues/28774). [#28965](https://github.com/ClickHouse/ClickHouse/pull/28965) ([小路](https://github.com/nicelulu)). +* We need to implement similar commands in clickhouse-keeper: https://zookeeper.apache.org/doc/r3.4.8/zookeeperAdmin.html#sc_zkCommands. [#28981](https://github.com/ClickHouse/ClickHouse/pull/28981) ([JackyWoo](https://github.com/JackyWoo)). +* Add option to compress logs before writing them to a file using LZ4. Closes [#23860](https://github.com/ClickHouse/ClickHouse/issues/23860). [#29219](https://github.com/ClickHouse/ClickHouse/pull/29219) ([Nikolay Degterinsky](https://github.com/evillique)). +* Introduced window functions: - `exponentialTimeDecayedSum` - `exponentialTimeDecayedMax` - `exponentialTimeDecayedCount` - `exponentialTimeDecayedAvg` which are more effective than `exponentialMovingAverage` for bigger windows. Also more use-cases were covered. [#29799](https://github.com/ClickHouse/ClickHouse/pull/29799) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Support for PARTITION BY in File, URL, HDFS storages and with INSERT INTO table function. Closes [#30273](https://github.com/ClickHouse/ClickHouse/issues/30273). [#30690](https://github.com/ClickHouse/ClickHouse/pull/30690) ([Kseniia Sumarokova](https://github.com/kssenii)). +* support bool data type. [#31072](https://github.com/ClickHouse/ClickHouse/pull/31072) ([kevin wan](https://github.com/MaxWk)). +* Exposes all GlobalThreadPool configurations to the configuration files. [#31285](https://github.com/ClickHouse/ClickHouse/pull/31285) ([Tomáš Hromada](https://github.com/gyfis)). +* Aliyun OSS Storage support. [#31286](https://github.com/ClickHouse/ClickHouse/pull/31286) ([cfcz48](https://github.com/cfcz48)). +* Allow to print/parse names and types of colums in CustomSeparated input/output format. Add formats CustomSeparatedWithNames/WithNamesAndTypes similar to TSVWithNames/WithNamesAndTypes. [#31434](https://github.com/ClickHouse/ClickHouse/pull/31434) ([Kruglov Pavel](https://github.com/Avogar)). +* - Basic access authentication for http/url functions. [#31648](https://github.com/ClickHouse/ClickHouse/pull/31648) ([michael1589](https://github.com/michael1589)). + +#### Performance Improvement +* ... Allow to split GraphiteMergeTree rollup rules for plain/tagged metrics (optional rule_type field). [#25122](https://github.com/ClickHouse/ClickHouse/pull/25122) ([Michail Safronov](https://github.com/msaf1980)). +* Fixing query performance issue in `LiveView` tables. Fixes [#30831](https://github.com/ClickHouse/ClickHouse/issues/30831). [#31006](https://github.com/ClickHouse/ClickHouse/pull/31006) ([vzakaznikov](https://github.com/vzakaznikov)). +* Improve performance of syncing data to block device. This closes [#31181](https://github.com/ClickHouse/ClickHouse/issues/31181). [#31229](https://github.com/ClickHouse/ClickHouse/pull/31229) ([zhanglistar](https://github.com/zhanglistar)). +* Support parallel formatting for all text formats, except `JSONEachRowWithProgress` and `PrettyCompactMonoBlock`. [#31489](https://github.com/ClickHouse/ClickHouse/pull/31489) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve performance of JSON and XML output formats. [#31673](https://github.com/ClickHouse/ClickHouse/pull/31673) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Speedup avg and sumCount aggregate functions. [#31694](https://github.com/ClickHouse/ClickHouse/pull/31694) ([Raúl Marín](https://github.com/Algunenano)). +* Speed up count over nullable columns. [#31806](https://github.com/ClickHouse/ClickHouse/pull/31806) ([Raúl Marín](https://github.com/Algunenano)). +* Speedup query parsing. [#31949](https://github.com/ClickHouse/ClickHouse/pull/31949) ([Raúl Marín](https://github.com/Algunenano)). + +#### Improvement +* Enable clang `-fstrict-vtable-pointers`, `-fwhole-program-vtables` compile options. [#20151](https://github.com/ClickHouse/ClickHouse/pull/20151) ([Maksim Kita](https://github.com/kitaisreal)). +* Skipping mutations of different partitions in `StorageMergeTree`. [#21326](https://github.com/ClickHouse/ClickHouse/pull/21326) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Closes [#12552](https://github.com/ClickHouse/ClickHouse/issues/12552). Allow versioning of aggregate function states. [#24820](https://github.com/ClickHouse/ClickHouse/pull/24820) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add optimizations for constant conditions in JOIN ON, ref [#26928](https://github.com/ClickHouse/ClickHouse/issues/26928). [#27021](https://github.com/ClickHouse/ClickHouse/pull/27021) ([Vladimir C](https://github.com/vdimir)). +* Add support for `Identifier` table and database query parameters. Closes [#27226](https://github.com/ClickHouse/ClickHouse/issues/27226). [#28668](https://github.com/ClickHouse/ClickHouse/pull/28668) ([Nikolay Degterinsky](https://github.com/evillique)). +* Allow to specify one or any number of PostgreSQL schemas for one MaterializedPostgreSQL database. Closes [#28901](https://github.com/ClickHouse/ClickHouse/issues/28901). Closes [#29324](https://github.com/ClickHouse/ClickHouse/issues/29324). [#28933](https://github.com/ClickHouse/ClickHouse/pull/28933) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make reading from HTTP retriable. Closes [#29696](https://github.com/ClickHouse/ClickHouse/issues/29696). [#29894](https://github.com/ClickHouse/ClickHouse/pull/29894) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add support for parallel reading from multiple files and support globs in `FROM INFILE` clause. [#30135](https://github.com/ClickHouse/ClickHouse/pull/30135) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* - Refactor formats TSV, TSVRaw, CSV and JSONCompactEachRow, JSONCompactStringsEachRow, remove code duplication, add base interface for formats with -WithNames and -WithNamesAndTypes suffixes. - Add formats CSVWithNamesAndTypes, TSVRawWithNames, TSVRawWithNamesAndTypes, JSONCompactEachRowWIthNames, JSONCompactStringsEachRowWIthNames, RowBinaryWithNames - Support parallel parsing for formats TSVWithNamesAndTypes, TSVRaw(WithNames/WIthNamesAndTypes), CSVWithNamesAndTypes, JSONCompactEachRow(WithNames/WIthNamesAndTypes), JSONCompactStringsEachRow(WithNames/WIthNamesAndTypes). - Support columns mapping and types checking for RowBinaryWithNamesAndTypes format. - Add setting `input_format_with_types_use_header` which specify if we should check that types written in WIthNamesAndTypes format matches with table structure. - Add setting `input_format_csv_empty_as_default` and use it in CSV format instead of `input_format_defaults_for_omitted_fields` (because this setting should't control `csv_empty_as_default`). - Fix usage of setting `input_format_defaults_for_omitted_fields` (it was used only as `csv_empty_as_default`, but it should control calculation of default expressions for omitted fields) - Fix Nullable input/output in TSVRaw format, make this format fully compatible with inserting into TSV. - Fix inserting NULLs in LowCardinality(Nullable) when `input_format_null_as_default` is enabled (previously default values was inserted instead of actual NULLs). - Fix strings deserialization in JSONStringsEachRow/JSONCompactStringsEachRow formats (strings were parsed just until first '\n' or '\t') - Add ability to use `Raw` escaping rule in Template input format. - Add diagnostic info for JSONCompactEachRow(WithNames/WIthNamesAndTypes) input format. - Fix bug with parallel parsing of -WithNames formats in case when setting min_chunk_bytes_for_parallel_parsing is less than bytes in a single row. [#30178](https://github.com/ClickHouse/ClickHouse/pull/30178) ([Kruglov Pavel](https://github.com/Avogar)). +* Avro format works against Kafka. Setting `output_format_avro_rows_in_file` added. [#30351](https://github.com/ClickHouse/ClickHouse/pull/30351) ([Ilya Golshtein](https://github.com/ilejn)). +* Implement the commands BACKUP and RESTORE for the Log family. [#30688](https://github.com/ClickHouse/ClickHouse/pull/30688) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix possible "The local set of parts of X doesn't look like the set of parts in ZooKeeper" error (if DROP fails during removing znodes from zookeeper). [#30826](https://github.com/ClickHouse/ClickHouse/pull/30826) ([Azat Khuzhin](https://github.com/azat)). +* For clickhouse-local or clickhouse-client if there is --interactive option with --query or --queries-file, then first execute them like in non-interactive and then start interactive mode. [#30851](https://github.com/ClickHouse/ClickHouse/pull/30851) ([Kseniia Sumarokova](https://github.com/kssenii)). +* added \l, \d, \c aliases like in MySQL. [#30876](https://github.com/ClickHouse/ClickHouse/pull/30876) ([Pavel Medvedev](https://github.com/pmed)). +* Fix `--verbose` option in clickhouse-local interactive mode and allow logging into file. [#30881](https://github.com/ClickHouse/ClickHouse/pull/30881) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Support `INTERVAL` type in `STEP` clause for `WITH FILL` modifier. [#30927](https://github.com/ClickHouse/ClickHouse/pull/30927) ([Anton Popov](https://github.com/CurtizJ)). +* Reduce memory usage when reading with `s3` / `url` / `hdfs` formats `Parquet`, `ORC`, `Arrow` (controlled by setting `input_format_allow_seeks`, enabled by default). Also add setting `remote_read_min_bytes_for_seek` to control seeks. Closes [#10461](https://github.com/ClickHouse/ClickHouse/issues/10461). Closes [#16857](https://github.com/ClickHouse/ClickHouse/issues/16857). [#30936](https://github.com/ClickHouse/ClickHouse/pull/30936) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add settings `merge_tree_min_rows_for_concurrent_read_for_remote_filesystem` and `merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem`. [#30970](https://github.com/ClickHouse/ClickHouse/pull/30970) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Do not allow to drop a table or dictionary if some tables or dictionaries depend on it. [#30977](https://github.com/ClickHouse/ClickHouse/pull/30977) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Only grab AlterLock when we do alter command. Let's see if the assumption is correct. [#31010](https://github.com/ClickHouse/ClickHouse/pull/31010) ([Amos Bird](https://github.com/amosbird)). +* The local session inside a Clickhouse dictionary source won't send its events to the session log anymore. This fixes a possible deadlock (tsan alert) on shutdown. Also this PR fixes flaky `test_dictionaries_dependency_xml/`. [#31013](https://github.com/ClickHouse/ClickHouse/pull/31013) ([Vitaly Baranov](https://github.com/vitlibar)). +* Cancel vertical merges when partition is dropped. This is a follow-up of https://github.com/ClickHouse/ClickHouse/pull/25684 and https://github.com/ClickHouse/ClickHouse/pull/30996. [#31057](https://github.com/ClickHouse/ClickHouse/pull/31057) ([Amos Bird](https://github.com/amosbird)). +* Support `IF EXISTS` modifier for `RENAME DATABASE`/`TABLE`/`DICTIONARY` query, If this directive is used, one will not get an error if the DATABASE/TABLE/DICTIONARY to be renamed doesn't exist. [#31081](https://github.com/ClickHouse/ClickHouse/pull/31081) ([victorgao](https://github.com/kafka1991)). +* Function name normalization for ALTER queries. This helps avoid metadata mismatch between creating table with indices/projections and adding indices/projections via alter commands. This is a follow-up PR of https://github.com/ClickHouse/ClickHouse/pull/20174. Mark as improvements as there are no bug reports and the senario is somehow rare. [#31095](https://github.com/ClickHouse/ClickHouse/pull/31095) ([Amos Bird](https://github.com/amosbird)). +* Enable multiline editing in clickhouse-client by default. This addresses [#31121](https://github.com/ClickHouse/ClickHouse/issues/31121) . [#31123](https://github.com/ClickHouse/ClickHouse/pull/31123) ([Amos Bird](https://github.com/amosbird)). +* Use DiskPtr instead of OS's file system API in class IDiskRemote in order to get more extendiability. Closes [#31117](https://github.com/ClickHouse/ClickHouse/issues/31117). [#31136](https://github.com/ClickHouse/ClickHouse/pull/31136) ([Yangkuan Liu](https://github.com/LiuYangkuan)). +* Now every replica will send to client only incremental information about profile events counters. [#31155](https://github.com/ClickHouse/ClickHouse/pull/31155) ([Dmitry Novik](https://github.com/novikd)). +* - Syntax changed so now backup engine should be set explicitly: `BACKUP ... TO Disk('backups', 'path\')` - Changed the format of backup's metadata, now it's in XML - Backup of a whole database now works. [#31178](https://github.com/ClickHouse/ClickHouse/pull/31178) ([Vitaly Baranov](https://github.com/vitlibar)). +* Improved backoff for background cleanup tasks in `MergeTree`. Settings `merge_tree_clear_old_temporary_directories_interval_seconds` and `merge_tree_clear_old_parts_interval_seconds` moved form users settings to merge tree settings. [#31180](https://github.com/ClickHouse/ClickHouse/pull/31180) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Optimize function `mapContains` to reading of subcolumn `key` with enabled settings `optimize_functions_to_subcolumns`. [#31218](https://github.com/ClickHouse/ClickHouse/pull/31218) ([Anton Popov](https://github.com/CurtizJ)). +* If some obsolete setting is changed show warning in `system.warnings`. [#31252](https://github.com/ClickHouse/ClickHouse/pull/31252) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Optimize function `tupleElement` to reading of subcolumn with enabled setting `optimize_functions_to_subcolumns`. [#31261](https://github.com/ClickHouse/ClickHouse/pull/31261) ([Anton Popov](https://github.com/CurtizJ)). +* Initial user's roles are used now to find row policies, see [#31080](https://github.com/ClickHouse/ClickHouse/issues/31080). [#31262](https://github.com/ClickHouse/ClickHouse/pull/31262) ([Vitaly Baranov](https://github.com/vitlibar)). +* Previously progress was shown only for `numbers` table function, not for `numbers_mt`. Now for `numbers_mt` it is also shown. [#31318](https://github.com/ClickHouse/ClickHouse/pull/31318) ([Kseniia Sumarokova](https://github.com/kssenii)). +* return fake create query when executing `show create table` on system's tables. [#31391](https://github.com/ClickHouse/ClickHouse/pull/31391) ([SuperDJY](https://github.com/cmsxbc)). +* MaterializedMySQL now handles `CREATE TABLE ... LIKE ...` DDL queries. [#31410](https://github.com/ClickHouse/ClickHouse/pull/31410) ([Stig Bakken](https://github.com/stigsb)). +* Default value of `http_send_timeout` and `http_receive_timeout` settings changed from 1800 (30 minutes) to 180 (3 minutes). [#31450](https://github.com/ClickHouse/ClickHouse/pull/31450) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Throw an exception if there is some garbage after field in JSONCompactStrings(EachRow) format. [#31455](https://github.com/ClickHouse/ClickHouse/pull/31455) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix waiting of the editor during interactive query edition (`waitpid()` returns -1 on `SIGWINCH` and `EDITOR` and `clickhouse-local`/`clickhouse-client` works concurrently). [#31456](https://github.com/ClickHouse/ClickHouse/pull/31456) ([Azat Khuzhin](https://github.com/azat)). +* Add `--pager` support for `clickhouse-local`. [#31457](https://github.com/ClickHouse/ClickHouse/pull/31457) ([Azat Khuzhin](https://github.com/azat)). +* Better analysis for `min/max/count` projection. Now, with enabled `allow_experimental_projection_optimization`, virtual `min/max/count` projection can be used together with columns from partition key. [#31474](https://github.com/ClickHouse/ClickHouse/pull/31474) ([Amos Bird](https://github.com/amosbird)). +* Use shard and replica name from `Replicated` database arguments when expanding macros in `ReplicatedMergeTree` arguments if these macros are not defined in config. Closes [#31471](https://github.com/ClickHouse/ClickHouse/issues/31471). [#31488](https://github.com/ClickHouse/ClickHouse/pull/31488) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better exception message when `users.xml` cannot be loaded due to bad password hash. This closes [#24126](https://github.com/ClickHouse/ClickHouse/issues/24126). [#31557](https://github.com/ClickHouse/ClickHouse/pull/31557) ([Vitaly Baranov](https://github.com/vitlibar)). +* Improve the `max_execution_time` checks. Fixed some cases when timeout checks do not happen and query could run too long. [#31636](https://github.com/ClickHouse/ClickHouse/pull/31636) ([Raúl Marín](https://github.com/Algunenano)). +* Add bindings for navigating through history (instead of lines/history). [#31641](https://github.com/ClickHouse/ClickHouse/pull/31641) ([Azat Khuzhin](https://github.com/azat)). +* Always re-render prompt while navigating history in clickhouse-client. This will improve usability of manipulating very long queries that don't fit on screen. [#31675](https://github.com/ClickHouse/ClickHouse/pull/31675) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to use named collections configuration for kafka and rabbitmq engines (the same way as for other intgration table engines). [#31691](https://github.com/ClickHouse/ClickHouse/pull/31691) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ClickHouse dictionary source support named connections. Closes [#31705](https://github.com/ClickHouse/ClickHouse/issues/31705). [#31749](https://github.com/ClickHouse/ClickHouse/pull/31749) ([Kseniia Sumarokova](https://github.com/kssenii)). +* MaterializedMySQL: Fix issue with table named 'table'. [#31781](https://github.com/ClickHouse/ClickHouse/pull/31781) ([Håvard Kvålen](https://github.com/havardk)). +* Recreate system.*_log tables in case of different engine/partition_by. [#31824](https://github.com/ClickHouse/ClickHouse/pull/31824) ([Azat Khuzhin](https://github.com/azat)). +* Fix the issue that `LowCardinality` of `Int256` cannot be created. [#31832](https://github.com/ClickHouse/ClickHouse/pull/31832) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support PostgreSQL style ALTER MODIFY COLUMN. [#32003](https://github.com/ClickHouse/ClickHouse/pull/32003) ([SuperDJY](https://github.com/cmsxbc)). +* Remove excessive `DESC TABLE` requests for `remote()` (in case of `remote('127.1', system.one)` (i.e. identifier as the db.table instead of string) there was excessive `DESC TABLE` request). [#32019](https://github.com/ClickHouse/ClickHouse/pull/32019) ([Azat Khuzhin](https://github.com/azat)). +* - Fix a bug that opentelemetry span log duration is zero at the query level if there's query exception. [#32038](https://github.com/ClickHouse/ClickHouse/pull/32038) ([Frank Chen](https://github.com/FrankChen021)). +* Added ClickHouse `exception` and `exception_code` fields to opentelemetry span log. [#32040](https://github.com/ClickHouse/ClickHouse/pull/32040) ([Frank Chen](https://github.com/FrankChen021)). +* Allow a user configured `hdfs_replication` parameter for DiskHdfs and StorageHdfs. Closes [#32039](https://github.com/ClickHouse/ClickHouse/issues/32039). [#32049](https://github.com/ClickHouse/ClickHouse/pull/32049) ([leosunli](https://github.com/leosunli)). +* Allow to write `+` before Float32/Float64 values. [#32079](https://github.com/ClickHouse/ClickHouse/pull/32079) ([Kruglov Pavel](https://github.com/Avogar)). +* - returns Content-Type as 'application/json' for `JSONEachRow` format if `output_format_json_array_of_rows` is enabled. [#32112](https://github.com/ClickHouse/ClickHouse/pull/32112) ([Frank Chen](https://github.com/FrankChen021)). +* - Set Content-Type in HTTP packets issued from URL engine. [#32113](https://github.com/ClickHouse/ClickHouse/pull/32113) ([Frank Chen](https://github.com/FrankChen021)). +* Now `clickhouse-keeper` refuse to start or apply configuration changes when they contain duplicated IDs or endpoints. Fixes [#31339](https://github.com/ClickHouse/ClickHouse/issues/31339). [#32121](https://github.com/ClickHouse/ClickHouse/pull/32121) ([alesapin](https://github.com/alesapin)). +* Added `update_field` support for `RangeHashedDictionary`, `ComplexKeyRangeHashedDictionary`. [#32185](https://github.com/ClickHouse/ClickHouse/pull/32185) ([Maksim Kita](https://github.com/kitaisreal)). +* Improve skiping unknown fields with Quoted escaping rule in Template/CustomSeparated formats. Previously we could skip only quoted strings, now we can skip values with any type. [#32204](https://github.com/ClickHouse/ClickHouse/pull/32204) ([Kruglov Pavel](https://github.com/Avogar)). +* Use `Content-Type: application/x-ndjson` (http://ndjson.org/) for output format `JSONEachRow`. [#32223](https://github.com/ClickHouse/ClickHouse/pull/32223) ([Dmitriy Dorofeev](https://github.com/deem0n)). +* - Improve the operation name of an opentelemetry span. [#32234](https://github.com/ClickHouse/ClickHouse/pull/32234) ([Frank Chen](https://github.com/FrankChen021)). +* Support default expression for storage hdfs and optimize fetching when source is column oriented. [#32256](https://github.com/ClickHouse/ClickHouse/pull/32256) ([李扬](https://github.com/taiyang-li)). + +#### Bug Fix +* Memory amount was incorrectly estimated when ClickHouse is run in containers with cgroup limits. [#31157](https://github.com/ClickHouse/ClickHouse/pull/31157) ([Pavel Medvedev](https://github.com/pmed)). +* Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). +* Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). +* Fix skipping columns while writing protobuf. This PR fixes [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160), see the comment [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160)#issuecomment-980595318. [#31988](https://github.com/ClickHouse/ClickHouse/pull/31988) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). + +#### Build/Testing/Packaging Improvement +* Hermetic builds: use fixed version of libc and make sure that no source or binary files from the host OS are using during build. This closes [#27133](https://github.com/ClickHouse/ClickHouse/issues/27133). This closes [#21435](https://github.com/ClickHouse/ClickHouse/issues/21435). This closes [#30462](https://github.com/ClickHouse/ClickHouse/issues/30462). [#30011](https://github.com/ClickHouse/ClickHouse/pull/30011) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use our own CMakeLists for `zlib-ng`, `cassandra`, `mariadb-connector-c` and `xz`, `re2`, `sentry`, `gsasl`, `arrow`, `protobuf`. This is needed for [#20151](https://github.com/ClickHouse/ClickHouse/issues/20151). Part of [#9226](https://github.com/ClickHouse/ClickHouse/issues/9226). A small step towards removal of annoying trash from the build system. [#30599](https://github.com/ClickHouse/ClickHouse/pull/30599) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build snappy error in [#30790](https://github.com/ClickHouse/ClickHouse/issues/30790) Update of contrib/snappy is in https://github.com/google/snappy/pull/145/files. [#30796](https://github.com/ClickHouse/ClickHouse/pull/30796) ([李扬](https://github.com/taiyang-li)). +* Drop support for using Ordinary databases with MaterializedMySQL. [#31292](https://github.com/ClickHouse/ClickHouse/pull/31292) ([Stig Bakken](https://github.com/stigsb)). +* Initial support for risc-v. See development/build-cross-riscv for quirks and build command that was tested. [#31309](https://github.com/ClickHouse/ClickHouse/pull/31309) ([Vladimir Smirnov](https://github.com/Civil)). +* Remove hardcoded repository name from CI scripts. [#31536](https://github.com/ClickHouse/ClickHouse/pull/31536) ([Constantine Peresypkin](https://github.com/pkit)). +* Avoid downloading toolchain tarballs for cross-compiling for FreeBSD. [#31672](https://github.com/ClickHouse/ClickHouse/pull/31672) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The script for uploading packages to the artifactory is added. [#31748](https://github.com/ClickHouse/ClickHouse/pull/31748) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Replaced default ports for clickhouse-keeper internal communication from 44444 to 9234. Fixes [#30879](https://github.com/ClickHouse/ClickHouse/issues/30879). [#31799](https://github.com/ClickHouse/ClickHouse/pull/31799) ([alesapin](https://github.com/alesapin)). +* More correct setting up capabilities inside Docker. [#31802](https://github.com/ClickHouse/ClickHouse/pull/31802) ([Constantine Peresypkin](https://github.com/pkit)). +* Revert changes from [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016): archive.ubuntu.com should be faster in general than RU mirror. [#31822](https://github.com/ClickHouse/ClickHouse/pull/31822) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove filesystem path to the build directory from binaries to enable reproducible builds. This needed for [#22113](https://github.com/ClickHouse/ClickHouse/issues/22113). [#31838](https://github.com/ClickHouse/ClickHouse/pull/31838) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make ClickHouse build fully reproducible (byte identical on different machines). This closes [#22113](https://github.com/ClickHouse/ClickHouse/issues/22113). [#31899](https://github.com/ClickHouse/ClickHouse/pull/31899) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* - Adjust artifactory pusher to a new bucket paths - Use only version or pull request number in bucket, no `0` - Create a function to read github event data. [#31952](https://github.com/ClickHouse/ClickHouse/pull/31952) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Build rpm and tgz packages in master and release branches workfolw. [#32048](https://github.com/ClickHouse/ClickHouse/pull/32048) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix broken symlink for sysroot/linux-riscv64/usr/lib. [#32071](https://github.com/ClickHouse/ClickHouse/pull/32071) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Fix some corner cases with intersect/except. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)). +* Fix bug which broke select queries if they happened after dropping materialized view. Found in [#30691](https://github.com/ClickHouse/ClickHouse/issues/30691). [#30997](https://github.com/ClickHouse/ClickHouse/pull/30997) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix JSONValue/Query with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible assert in `hdfs` table function/engine, add test. [#31036](https://github.com/ClickHouse/ClickHouse/pull/31036) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix StorageMerge with aliases and where (it did not work before at all). Closes [#28802](https://github.com/ClickHouse/ClickHouse/issues/28802). [#31044](https://github.com/ClickHouse/ClickHouse/pull/31044) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Rewrite right distributed table in local join. solves [#25809](https://github.com/ClickHouse/ClickHouse/issues/25809). [#31105](https://github.com/ClickHouse/ClickHouse/pull/31105) ([abel-cheng](https://github.com/abel-cheng)). +* Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)). +* Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)). +* Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix bug with group by and positional arguments. Closes [#31280](https://github.com/ClickHouse/ClickHouse/issues/31280)#issuecomment-968696186. [#31420](https://github.com/ClickHouse/ClickHouse/pull/31420) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix progress for short INSERT SELECT queries. [#31510](https://github.com/ClickHouse/ClickHouse/pull/31510) ([Azat Khuzhin](https://github.com/azat)). +* * Disable `partial_merge_join_left_table_buffer_bytes` before bug in this optimization is fixed. See [#31009](https://github.com/ClickHouse/ClickHouse/issues/31009)). * Remove redundant option `partial_merge_join_optimizations`. [#31528](https://github.com/ClickHouse/ClickHouse/pull/31528) ([Vladimir C](https://github.com/vdimir)). +* Fix invalid generated JSON when only column names contain invalid UTF-8 sequences. [#31534](https://github.com/ClickHouse/ClickHouse/pull/31534) ([Kevin Michel](https://github.com/kmichel-aiven)). +* All non-x86 builds were broken, because we don't have tests for them. This closes [#31417](https://github.com/ClickHouse/ClickHouse/issues/31417). This closes [#31524](https://github.com/ClickHouse/ClickHouse/issues/31524). [#31574](https://github.com/ClickHouse/ClickHouse/pull/31574) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix sparkbars are not aligned, see: [#26175](https://github.com/ClickHouse/ClickHouse/issues/26175)#issuecomment-960353867, [comment](https://github.com/ClickHouse/ClickHouse/issues/26175#issuecomment-961155065). [#31624](https://github.com/ClickHouse/ClickHouse/pull/31624) ([小路](https://github.com/nicelulu)). +* `RENAME TABLE` query worked incorrectly on attempt to rename an DDL dictionary in `Ordinary` database, it's fixed. [#31638](https://github.com/ClickHouse/ClickHouse/pull/31638) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed null pointer exception in `MATERIALIZE COLUMN`. [#31679](https://github.com/ClickHouse/ClickHouse/pull/31679) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Settings `input_format_allow_errors_num` and `input_format_allow_errors_ratio` did not work for parsing of domain types, such as `IPv4`, it's fixed. Fixes [#31686](https://github.com/ClickHouse/ClickHouse/issues/31686). [#31697](https://github.com/ClickHouse/ClickHouse/pull/31697) ([Alexander Tokmakov](https://github.com/tavplubix)). +* * Fixed function ngrams when string contains utf8 characters. [#31706](https://github.com/ClickHouse/ClickHouse/pull/31706) ([yandd](https://github.com/yandd)). +* Fix exception on some of the applications of `decrypt` function on Nullable columns. This closes [#31662](https://github.com/ClickHouse/ClickHouse/issues/31662). This closes [#31426](https://github.com/ClickHouse/ClickHouse/issues/31426). [#31707](https://github.com/ClickHouse/ClickHouse/pull/31707) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed `there are no such cluster here` error on execution of `ON CLUSTER` query if specified cluster name is name of `Replicated` database. [#31723](https://github.com/ClickHouse/ClickHouse/pull/31723) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix race in JSONEachRowWithProgress output format when data and lines with progress are mixed in output. [#31736](https://github.com/ClickHouse/ClickHouse/pull/31736) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixed rare segfault on concurrent `ATTACH PARTITION` queries. [#31738](https://github.com/ClickHouse/ClickHouse/pull/31738) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix disabling query profiler (In case of `query_profiler_real_time_period_ns>0`/`query_profiler_cpu_time_period_ns>0` query profiler can stayed enabled even after query finished). [#31740](https://github.com/ClickHouse/ClickHouse/pull/31740) ([Azat Khuzhin](https://github.com/azat)). +* Fix group by / order by / limit by aliases with positional arguments enabled. Closes [#31173](https://github.com/ClickHouse/ClickHouse/issues/31173). [#31741](https://github.com/ClickHouse/ClickHouse/pull/31741) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix usage of `Buffer` table engine with type `Map`. Fixes [#30546](https://github.com/ClickHouse/ClickHouse/issues/30546). [#31742](https://github.com/ClickHouse/ClickHouse/pull/31742) ([Anton Popov](https://github.com/CurtizJ)). +* Fix crash with empty result on odbc query. Closes [#31465](https://github.com/ClickHouse/ClickHouse/issues/31465). [#31766](https://github.com/ClickHouse/ClickHouse/pull/31766) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix recursive user defined functions crash. Closes [#30856](https://github.com/ClickHouse/ClickHouse/issues/30856). [#31820](https://github.com/ClickHouse/ClickHouse/pull/31820) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix invalid cast of nullable type when nullable primary key is used. This fixes [#31075](https://github.com/ClickHouse/ClickHouse/issues/31075). [#31823](https://github.com/ClickHouse/ClickHouse/pull/31823) ([Amos Bird](https://github.com/amosbird)). +* Fix reading from `MergeTree` tables with enabled `use_uncompressed_cache`. [#31826](https://github.com/ClickHouse/ClickHouse/pull/31826) ([Anton Popov](https://github.com/CurtizJ)). +* Fix a bug about function transform with decimal args. [#31839](https://github.com/ClickHouse/ClickHouse/pull/31839) ([Shuai li](https://github.com/loneylee)). +* - Change configuration path from `keeper_server.session_timeout_ms` to `keeper_server.coordination_settings.session_timeout_ms` when constructing a `KeeperTCPHandler` - Same with `operation_timeout`. [#31859](https://github.com/ClickHouse/ClickHouse/pull/31859) ([JackyWoo](https://github.com/JackyWoo)). +* Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +* Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix FileLog engine unnesessary create meta data directory when create table failed. Fix [#31962](https://github.com/ClickHouse/ClickHouse/issues/31962). [#31967](https://github.com/ClickHouse/ClickHouse/pull/31967) ([flynn](https://github.com/ucasfl)). +* MaterializedMySQL: Fix rare corruption of DECIMAL data. [#31990](https://github.com/ClickHouse/ClickHouse/pull/31990) ([Håvard Kvålen](https://github.com/havardk)). +* Fixed `Directory ... already exists and is not empty` error when detaching part. [#32063](https://github.com/ClickHouse/ClickHouse/pull/32063) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix CREATE TABLE of Join Storage with multiply settings contains persistency. Close [#31680](https://github.com/ClickHouse/ClickHouse/issues/31680). [#32066](https://github.com/ClickHouse/ClickHouse/pull/32066) ([SuperDJY](https://github.com/cmsxbc)). +* Fix `CAST` from `Nullable` with `cast_keep_nullable` (`PARAMETER_OUT_OF_BOUND` error before for i.e. `toUInt32OrDefault(toNullable(toUInt32(1)))`). [#32080](https://github.com/ClickHouse/ClickHouse/pull/32080) ([Azat Khuzhin](https://github.com/azat)). +* Dictionaries fix cases when `{condition}` does not work for custom database queries. [#32117](https://github.com/ClickHouse/ClickHouse/pull/32117) ([Maksim Kita](https://github.com/kitaisreal)). +* Number of active replicas might be determined incorrectly when inserting with quorum if setting `replicated_can_become_leader` is disabled on some replicas. It's fixed. [#32157](https://github.com/ClickHouse/ClickHouse/pull/32157) ([Alexander Tokmakov](https://github.com/tavplubix)). +* XML dictionaries identifiers, used in table create query, can be qualified to `default_database` during upgrade to newer version. Closes [#31963](https://github.com/ClickHouse/ClickHouse/issues/31963). [#32187](https://github.com/ClickHouse/ClickHouse/pull/32187) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix parsing error while NaN deserializing for `Nullable(Float)` for `Quoted` escaping rule. [#32190](https://github.com/ClickHouse/ClickHouse/pull/32190) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix window view parser. [#32232](https://github.com/ClickHouse/ClickHouse/pull/32232) ([vxider](https://github.com/Vxider)). +* Server might fail to start with `Cannot attach 1 tables due to cyclic dependencies` error if `Dictionary` table looks at XML-dictionary with the same name, it's fixed. Fixes [#31315](https://github.com/ClickHouse/ClickHouse/issues/31315). [#32288](https://github.com/ClickHouse/ClickHouse/pull/32288) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed crash with SIGFPE in aggregate function `avgWeighted` with `Decimal` argument. Fixes [#32053](https://github.com/ClickHouse/ClickHouse/issues/32053). [#32303](https://github.com/ClickHouse/ClickHouse/pull/32303) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix `ALTER ... MATERIALIZE COLUMN ...` queries in case when data type of default expression is not equal to the data type of column. [#32348](https://github.com/ClickHouse/ClickHouse/pull/32348) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed the behavior when mutations that have nothing to do are stuck (with enabled setting `empty_result_for_aggregation_by_empty_set`). [#32358](https://github.com/ClickHouse/ClickHouse/pull/32358) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### Build + +* support compile in arm machine with parameter "-DENABLE_TESTS=OFF". [#31007](https://github.com/ClickHouse/ClickHouse/pull/31007) ([zhanghuajie](https://github.com/zhanghuajieHIT)). + +#### Improvement (changelog entry is not required) + +* Make remote_filesystem_read_method=threadpool by default. [#31291](https://github.com/ClickHouse/ClickHouse/pull/31291) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Imrovement (changelog entry is not required) + +* Rename setting value `read_threadpool` to `threadpool` for setting `remote_filesystem_read_method`. [#31224](https://github.com/ClickHouse/ClickHouse/pull/31224) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Update permissions-for-queries.md of cn'. [#30902](https://github.com/ClickHouse/ClickHouse/pull/30902) ([Laurie Li](https://github.com/laurieliyang)). +* NO CL ENTRY: 'Make use of untuple alias for untupled columns names prefix'. [#30984](https://github.com/ClickHouse/ClickHouse/pull/30984) ([qieqieplus](https://github.com/qieqieplus)). +* NO CL ENTRY: 'Add banner block for index,company,careers pages'. [#31647](https://github.com/ClickHouse/ClickHouse/pull/31647) ([Tom Risse](https://github.com/flickerbox-tom)). +* NO CL ENTRY: 'Revert "Fixed null pointer exception in `MATERIALIZE COLUMN`"'. [#31692](https://github.com/ClickHouse/ClickHouse/pull/31692) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Check time limit sending data for global in.'. [#31805](https://github.com/ClickHouse/ClickHouse/pull/31805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* NO CL ENTRY: 'Fix syntax error: drop comma'. [#32095](https://github.com/ClickHouse/ClickHouse/pull/32095) ([Federico Ceratto](https://github.com/FedericoCeratto)). +* NO CL ENTRY: 'Revert "Add a test with 20000 mutations in one query"'. [#32326](https://github.com/ClickHouse/ClickHouse/pull/32326) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* NO CL ENTRY: 'Revert "Revert "Add a test with 20000 mutations in one query""'. [#32327](https://github.com/ClickHouse/ClickHouse/pull/32327) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.12.2.17-stable.md b/docs/changelogs/v21.12.2.17-stable.md new file mode 100644 index 00000000000..909bc7917c7 --- /dev/null +++ b/docs/changelogs/v21.12.2.17-stable.md @@ -0,0 +1,22 @@ +### ClickHouse release v21.12.2.17-stable FIXME as compared to v21.12.1.9017-prestable + +#### Bug Fix +* Backported in [#32693](https://github.com/ClickHouse/ClickHouse/issues/32693): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#32681](https://github.com/ClickHouse/ClickHouse/issues/32681): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#32483](https://github.com/ClickHouse/ClickHouse/issues/32483): Fix 'APPLY lambda' parsing which could lead to client/server crash. [#32138](https://github.com/ClickHouse/ClickHouse/pull/32138) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#32542](https://github.com/ClickHouse/ClickHouse/issues/32542): Some replication queue entries might hang for `temporary_directories_lifetime` (1 day by default) with `Directory tmp_merge_` or `Part ... (state Deleting) already exists, but it will be deleted soon` or similar error. It's fixed. Fixes [#29616](https://github.com/ClickHouse/ClickHouse/issues/29616). [#32201](https://github.com/ClickHouse/ClickHouse/pull/32201) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32710](https://github.com/ClickHouse/ClickHouse/issues/32710): Fix failures in queries that are trying to use skipping indices, which are not materialized yet. Fixes [#32292](https://github.com/ClickHouse/ClickHouse/issues/32292) and [#30343](https://github.com/ClickHouse/ClickHouse/issues/30343). [#32359](https://github.com/ClickHouse/ClickHouse/pull/32359) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#32569](https://github.com/ClickHouse/ClickHouse/issues/32569): Fix crash in `JoinCommon::removeColumnNullability`, close [#32458](https://github.com/ClickHouse/ClickHouse/issues/32458). [#32508](https://github.com/ClickHouse/ClickHouse/pull/32508) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#32770](https://github.com/ClickHouse/ClickHouse/issues/32770): Fix sparse_hashed dict performance with sequential keys (wrong hash function). [#32536](https://github.com/ClickHouse/ClickHouse/pull/32536) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#32634](https://github.com/ClickHouse/ClickHouse/issues/32634): Fix table lifetime (i.e. possible use-after-free) in case of parallel DROP TABLE and INSERT. [#32572](https://github.com/ClickHouse/ClickHouse/pull/32572) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#32632](https://github.com/ClickHouse/ClickHouse/issues/32632): Fix possible exception at RabbitMQ storage startup by delaying channel creation. [#32584](https://github.com/ClickHouse/ClickHouse/pull/32584) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#32733](https://github.com/ClickHouse/ClickHouse/issues/32733): Fix surprisingly bad code in function `file`. [#32640](https://github.com/ClickHouse/ClickHouse/pull/32640) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#32793](https://github.com/ClickHouse/ClickHouse/issues/32793): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release): + +* Backported in [#32616](https://github.com/ClickHouse/ClickHouse/issues/32616): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.12.3.32-stable.md b/docs/changelogs/v21.12.3.32-stable.md new file mode 100644 index 00000000000..3c08aae4cba --- /dev/null +++ b/docs/changelogs/v21.12.3.32-stable.md @@ -0,0 +1,17 @@ +### ClickHouse release v21.12.3.32-stable FIXME as compared to v21.12.2.17-stable + +#### Bug Fix +* Backported in [#33018](https://github.com/ClickHouse/ClickHouse/issues/33018): - Clickhouse Keeper handler should remove operation when response sent. [#32988](https://github.com/ClickHouse/ClickHouse/pull/32988) ([JackyWoo](https://github.com/JackyWoo)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#32890](https://github.com/ClickHouse/ClickHouse/issues/32890): Fix LOGICAL_ERROR when the target of a materialized view is a JOIN or a SET table. [#32669](https://github.com/ClickHouse/ClickHouse/pull/32669) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#33183](https://github.com/ClickHouse/ClickHouse/issues/33183): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32904](https://github.com/ClickHouse/ClickHouse/issues/32904): `MergeTree` table engine might silently skip some mutations if there are too many running mutations or in case of high memory consumption, it's fixed. Fixes [#17882](https://github.com/ClickHouse/ClickHouse/issues/17882). [#32814](https://github.com/ClickHouse/ClickHouse/pull/32814) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#33047](https://github.com/ClickHouse/ClickHouse/issues/33047): Fix optimization with lazy seek for async reads from remote fs. Closes [#32803](https://github.com/ClickHouse/ClickHouse/issues/32803). [#32835](https://github.com/ClickHouse/ClickHouse/pull/32835) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#32932](https://github.com/ClickHouse/ClickHouse/issues/32932): Close [#32487](https://github.com/ClickHouse/ClickHouse/issues/32487). [#32914](https://github.com/ClickHouse/ClickHouse/pull/32914) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#32962](https://github.com/ClickHouse/ClickHouse/issues/32962): Fix a regression in `replaceRegexpAll` function. The function worked incorrectly when matched substring was empty. This closes [#32777](https://github.com/ClickHouse/ClickHouse/issues/32777). This closes [#30245](https://github.com/ClickHouse/ClickHouse/issues/30245). [#32945](https://github.com/ClickHouse/ClickHouse/pull/32945) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#33067](https://github.com/ClickHouse/ClickHouse/issues/33067): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#33100](https://github.com/ClickHouse/ClickHouse/issues/33100): Fix Context leak in case of cancel_http_readonly_queries_on_client_close (i.e. leaking of external tables that had been uploaded the the server and other resources). [#32982](https://github.com/ClickHouse/ClickHouse/pull/32982) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#33123](https://github.com/ClickHouse/ClickHouse/issues/33123): Fix error `Invalid version for SerializationLowCardinality key column` in case of reading from `LowCardinality` column with `local_filesystem_read_prefetch` or `remote_filesystem_read_prefetch` enabled. [#33046](https://github.com/ClickHouse/ClickHouse/pull/33046) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.12.4.1-stable.md b/docs/changelogs/v21.12.4.1-stable.md new file mode 100644 index 00000000000..7c028592876 --- /dev/null +++ b/docs/changelogs/v21.12.4.1-stable.md @@ -0,0 +1,13 @@ +### ClickHouse release v21.12.4.1-stable FIXME as compared to v21.12.3.32-stable + +#### Improvement +* Backported in [#33792](https://github.com/ClickHouse/ClickHouse/issues/33792): Create parent directories in DiskS3::restoreFileOperations method. [#33730](https://github.com/ClickHouse/ClickHouse/pull/33730) ([ianton-ru](https://github.com/ianton-ru)). + +#### Bug Fix +* Backported in [#33551](https://github.com/ClickHouse/ClickHouse/issues/33551): Fix null pointer dereference in low cardinality data when deserializing LowCardinality data in the Native format. [#33021](https://github.com/ClickHouse/ClickHouse/pull/33021) ([Harry Lee](https://github.com/HarryLeeIBM)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#33537](https://github.com/ClickHouse/ClickHouse/issues/33537): Fix ORC stripe reading. [#32929](https://github.com/ClickHouse/ClickHouse/pull/32929) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). +* Backported in [#33654](https://github.com/ClickHouse/ClickHouse/issues/33654): Fix segfault in Avro that appears after the second insert into file. [#33566](https://github.com/ClickHouse/ClickHouse/pull/33566) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.2.1.5869-prestable.md b/docs/changelogs/v21.2.1.5869-prestable.md new file mode 100644 index 00000000000..43703bc13b9 --- /dev/null +++ b/docs/changelogs/v21.2.1.5869-prestable.md @@ -0,0 +1,156 @@ +### ClickHouse release v21.2.1.5869-prestable FIXME as compared to v21.1.1.5646-prestable + +#### Backward Incompatible Change +* - Fix memory tracking for `OPTIMIZE TABLE`/merges - Account query memory limits and sampling for `OPTIMIZE TABLE`/merges. [#18772](https://github.com/ClickHouse/ClickHouse/pull/18772) ([Azat Khuzhin](https://github.com/azat)). +* Forbid `lcm`/`gcd` for floats. [#19532](https://github.com/ClickHouse/ClickHouse/pull/19532) ([Azat Khuzhin](https://github.com/azat)). +* Bitwise functions (`bitAnd`, `bitOr`, etc) are forbidden for floating point arguments. Now you have to do explicit cast to integer. [#19853](https://github.com/ClickHouse/ClickHouse/pull/19853) ([Azat Khuzhin](https://github.com/azat)). + +#### New Feature +* add support for zstd long option for better compression of string columns to save space. [#17184](https://github.com/ClickHouse/ClickHouse/pull/17184) ([ygrek](https://github.com/ygrek)). +* - Added support of mapping LDAP group names, and attribute values in general, to local roles for users from ldap user directories. [#17211](https://github.com/ClickHouse/ClickHouse/pull/17211) ([Denis Glazachev](https://github.com/traceon)). +* Data type `Nested` now supports arbitrary levels of nesting. Introduced subcolumns of complex types, such as `size0` in `Array`, `null` in `Nullable`, names of `Tuple` elements, which can be read without reading of whole column. [#17310](https://github.com/ClickHouse/ClickHouse/pull/17310) ([Anton Popov](https://github.com/CurtizJ)). +* Add support of tuple argument to `argMin` and `argMax` functions. [#17359](https://github.com/ClickHouse/ClickHouse/pull/17359) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Added `Nullable` support for `FlatDictionary`, `HashedDictionary`, `ComplexKeyHashedDictionary`, `DirectDictionary`, `ComplexKeyDirectDictionary`, `RangeHashedDictionary`. [#18236](https://github.com/ClickHouse/ClickHouse/pull/18236) ([Maksim Kita](https://github.com/kitaisreal)). +* Disallow floating point column as partition key related to : [#18421](https://github.com/ClickHouse/ClickHouse/issues/18421)#event-4147046255. [#18464](https://github.com/ClickHouse/ClickHouse/pull/18464) ([hexiaoting](https://github.com/hexiaoting)). +* Add function decodeXMLComponent to decode characters for XML. ``` SELECT decodeXMLComponent('Hello,"world"!'); ``` [#17659](https://github.com/ClickHouse/ClickHouse/issues/17659). [#18542](https://github.com/ClickHouse/ClickHouse/pull/18542) ([nauta](https://github.com/nautaa)). +* Added PostgreSQL table engine (both select/insert, with support for multidimensional arrays), also as table function. Added PostgreSQL dictionary source. Added PostgreSQL database engine. [#18554](https://github.com/ClickHouse/ClickHouse/pull/18554) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add `SELECT ALL` syntax. closes [#18706](https://github.com/ClickHouse/ClickHouse/issues/18706). [#18723](https://github.com/ClickHouse/ClickHouse/pull/18723) ([flynn](https://github.com/ucasfl)). +* Add three functions for map data type: 1. mapContains(map, key) to check weather map.keys include the second parameter key. 2. mapKeys(map) return all the keys in Array format 3. mapValues(map) return all the values in Array format. [#18788](https://github.com/ClickHouse/ClickHouse/pull/18788) ([hexiaoting](https://github.com/hexiaoting)). +* Support MetaKey+Enter hotkey binding in play ui. [#19012](https://github.com/ClickHouse/ClickHouse/pull/19012) ([sundyli](https://github.com/sundy-li)). +* Function formatDateTime support the %Q modification to format date to quarter. ... [#19224](https://github.com/ClickHouse/ClickHouse/pull/19224) ([Jianmei Zhang](https://github.com/zhangjmruc)). +* ... [#19261](https://github.com/ClickHouse/ClickHouse/pull/19261) ([RegulusZ](https://github.com/RegulusZ)). +* Add factories' objects names, created during query, into system.query_log. Closes [#18495](https://github.com/ClickHouse/ClickHouse/issues/18495). [#19371](https://github.com/ClickHouse/ClickHouse/pull/19371) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add `sign` math function. [#19527](https://github.com/ClickHouse/ClickHouse/pull/19527) ([flynn](https://github.com/ucasfl)). +* Added functions `parseDateTimeBestEffortUSOrZero`, `parseDateTimeBestEffortUSOrNull`. [#19712](https://github.com/ClickHouse/ClickHouse/pull/19712) ([Maksim Kita](https://github.com/kitaisreal)). +* ... [#19764](https://github.com/ClickHouse/ClickHouse/pull/19764) ([emhlbmc](https://github.com/emhlbmc)). + +#### Performance Improvement +* Use a connection pool for S3 connections, controlled by the `s3_max_connections` settings. [#13405](https://github.com/ClickHouse/ClickHouse/pull/13405) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Rewrite `sumIf()` and `sum(if())` function to `countIf()` function when logically equivalent. [#17041](https://github.com/ClickHouse/ClickHouse/pull/17041) ([flynn](https://github.com/ucasfl)). +* Update libcxx and use unstable ABI to provide better performance. [#18914](https://github.com/ClickHouse/ClickHouse/pull/18914) ([Daniel Kutenin](https://github.com/danlark1)). +* Faster parts removal by lowering the number of `stat` syscalls. This returns the optimization that existed while ago. More safe interface of `IDisk`. This closes [#19065](https://github.com/ClickHouse/ClickHouse/issues/19065). [#19086](https://github.com/ClickHouse/ClickHouse/pull/19086) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Speed up aggregate function `sum`. Improvement only visible on synthetic benchmarks and not very practical. [#19216](https://github.com/ClickHouse/ClickHouse/pull/19216) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support splitting `Filter` step of query plan into `Expression + Filter` pair. Together with `Expression + Expression` merging optimization ([#17458](https://github.com/ClickHouse/ClickHouse/issues/17458)) it may delay execution for some expressions after `Filter` step. [#19253](https://github.com/ClickHouse/ClickHouse/pull/19253) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reduce lock contention for multiple layers of the Buffer engine. [#19379](https://github.com/ClickHouse/ClickHouse/pull/19379) ([Azat Khuzhin](https://github.com/azat)). +* Slightly improve server latency by removing access to configuration on every connection. [#19863](https://github.com/ClickHouse/ClickHouse/pull/19863) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Added support for `WITH ... [AND] [PERIODIC] REFRESH [interval_in_sec]` clause when creating `LIVE VIEW` tables. [#14822](https://github.com/ClickHouse/ClickHouse/pull/14822) ([vzakaznikov](https://github.com/vzakaznikov)). +* - Add optimize_alias_column_prediction (on by default), that will: * Respect aliased columns in WHERE during partition pruning and skipping data using secondary indexes * Respect aliased columns in WHERE for trivial count queries for optimize_trivial_count * Respect aliased columns in GROUP BY/ORDER BY for optimize_aggregation_in_order/optimize_read_in_order. [#16995](https://github.com/ClickHouse/ClickHouse/pull/16995) ([sundyli](https://github.com/sundy-li)). +* Updated AWS C++ SDK in order to utilize global regions in S3. [#17870](https://github.com/ClickHouse/ClickHouse/pull/17870) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Support insert into table function `cluster`, and for both table functions `remote` and `cluster`, support distributing data across nodes by specify sharding key. Close [#16752](https://github.com/ClickHouse/ClickHouse/issues/16752). [#18264](https://github.com/ClickHouse/ClickHouse/pull/18264) ([flynn](https://github.com/ucasfl)). +* Support `EXISTS VIEW` syntax. [#18552](https://github.com/ClickHouse/ClickHouse/pull/18552) ([Du Chuan](https://github.com/spongedu)). +* Update librdkafka to v1.6.0-RC2. Fixes [#18668](https://github.com/ClickHouse/ClickHouse/issues/18668). [#18671](https://github.com/ClickHouse/ClickHouse/pull/18671) ([filimonov](https://github.com/filimonov)). +* Allow CTE to be further aliased. Propagate CSE to subqueries in the same level when `enable_global_with_statement = 1`. This fixes [#17378](https://github.com/ClickHouse/ClickHouse/issues/17378) . This fixes https://github.com/ClickHouse/ClickHouse/pull/16575#issuecomment-753416235 . [#18684](https://github.com/ClickHouse/ClickHouse/pull/18684) ([Amos Bird](https://github.com/amosbird)). +* Add [UInt8, UInt16, UInt32, UInt64] arguments types support for bitmapTransform, bitmapSubsetInRange, bitmapSubsetLimit, bitmapContains functions. This closes [#18713](https://github.com/ClickHouse/ClickHouse/issues/18713). [#18791](https://github.com/ClickHouse/ClickHouse/pull/18791) ([sundyli](https://github.com/sundy-li)). +* Added prefix-based S3 endpoint settings. [#18812](https://github.com/ClickHouse/ClickHouse/pull/18812) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix issues with RIGHT and FULL JOIN of tables with aggregate function states. In previous versions exception about `cloneResized` method was thrown. [#18818](https://github.com/ClickHouse/ClickHouse/pull/18818) ([templarzq](https://github.com/templarzq)). +* - Check per-block checksum of the distributed batch on the sender before sending (without reading the file twice, the checksums will be verified while reading), this will avoid stuck of the INSERT on the receiver (on truncated .bin file on the sender) - Avoid reading .bin files twice for batched INSERT (it was required to calculate rows/bytes to take squashing into account, now this information included into the header, backward compatible is preserved). [#18853](https://github.com/ClickHouse/ClickHouse/pull/18853) ([Azat Khuzhin](https://github.com/azat)). +* Add `normalizeQueryKeepNames` and `normalizedQueryHashKeepNames` to normalize queries without masking long names with `?`. This helps better analyze complex query logs. [#18910](https://github.com/ClickHouse/ClickHouse/pull/18910) ([Amos Bird](https://github.com/amosbird)). +* Docker image: several improvements for clickhouse-server entrypoint. [#18954](https://github.com/ClickHouse/ClickHouse/pull/18954) ([filimonov](https://github.com/filimonov)). +* Fixed `PeekableReadBuffer: Memory limit exceed` error when inserting data with huge strings. Fixes [#18690](https://github.com/ClickHouse/ClickHouse/issues/18690). [#18979](https://github.com/ClickHouse/ClickHouse/pull/18979) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Explicitly set uid / gid of clickhouse user & group to the fixed values (101) in clickhouse-server images. [#19096](https://github.com/ClickHouse/ClickHouse/pull/19096) ([filimonov](https://github.com/filimonov)). +* The exception when function `bar` is called with certain NaN argument may be slightly misleading in previous versions. This fixes [#19088](https://github.com/ClickHouse/ClickHouse/issues/19088). [#19107](https://github.com/ClickHouse/ClickHouse/pull/19107) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow change `max_server_memory_usage` without restart. This closes [#18154](https://github.com/ClickHouse/ClickHouse/issues/18154). [#19186](https://github.com/ClickHouse/ClickHouse/pull/19186) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong alignment of values of `IPv4` data type in Pretty formats. They were aligned to the right, not to the left. This closes [#19184](https://github.com/ClickHouse/ClickHouse/issues/19184). [#19339](https://github.com/ClickHouse/ClickHouse/pull/19339) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow docker to be executed with arbitrary uid. [#19374](https://github.com/ClickHouse/ClickHouse/pull/19374) ([filimonov](https://github.com/filimonov)). +* Add metrics for MergeTree parts (Wide/Compact/InMemory) types. [#19381](https://github.com/ClickHouse/ClickHouse/pull/19381) ([Azat Khuzhin](https://github.com/azat)). +* Improve MySQL compatibility. [#19387](https://github.com/ClickHouse/ClickHouse/pull/19387) ([Daniil Kondratyev](https://github.com/dankondr)). +* Add `http_referer` field to `system.query_log`, `system.processes`, etc. This closes [#19389](https://github.com/ClickHouse/ClickHouse/issues/19389). [#19390](https://github.com/ClickHouse/ClickHouse/pull/19390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* `toIPv6` function parses `IPv4` addresses. [#19518](https://github.com/ClickHouse/ClickHouse/pull/19518) ([Bharat Nallan](https://github.com/bharatnc)). +* Support using the new location of `.debug` file. This fixes [#19348](https://github.com/ClickHouse/ClickHouse/issues/19348). [#19520](https://github.com/ClickHouse/ClickHouse/pull/19520) ([Amos Bird](https://github.com/amosbird)). +* Enable function length/empty/notEmpty for datatype map, which returns keys number in map. [#19530](https://github.com/ClickHouse/ClickHouse/pull/19530) ([李扬](https://github.com/taiyang-li)). +* Support constant result in function `multiIf`. [#19533](https://github.com/ClickHouse/ClickHouse/pull/19533) ([Maksim Kita](https://github.com/kitaisreal)). +* Add an option to disable validation of checksums on reading. Should never be used in production. Please do not expect any benefits in disabling it. It may only be used for experiments and benchmarks. The setting only applicable for tables of MergeTree family. Checksums are always validated for other table engines and when receiving data over network. In my observations there is no performance difference or it is less than 0.5%. [#19588](https://github.com/ClickHouse/ClickHouse/pull/19588) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dictionary better error message during attribute parsing. [#19678](https://github.com/ClickHouse/ClickHouse/pull/19678) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix rare `max_number_of_merges_with_ttl_in_pool` limit overrun (more merges with TTL can be assigned) for non-replicated MergeTree. [#19708](https://github.com/ClickHouse/ClickHouse/pull/19708) ([alesapin](https://github.com/alesapin)). +* Insuffiient arguments check in `positionCaseInsensitiveUTF8` function triggered address sanitizer. [#19720](https://github.com/ClickHouse/ClickHouse/pull/19720) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add separate pool for message brokers (RabbitMQ and Kafka). [#19722](https://github.com/ClickHouse/ClickHouse/pull/19722) ([Azat Khuzhin](https://github.com/azat)). +* In distributed queries if the setting `async_socket_for_remote` is enabled, it was possible to get stack overflow at least in debug build configuration if very deeply nested data type is used in table (e.g. `Array(Array(Array(...more...)))`). This fixes [#19108](https://github.com/ClickHouse/ClickHouse/issues/19108). This change introduces minor backward incompatibility: excessive parenthesis in type definitions no longer supported, example: `Array((UInt8))`. [#19736](https://github.com/ClickHouse/ClickHouse/pull/19736) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Table function `S3` will use global region if the region can't be determined exactly. This closes [#10998](https://github.com/ClickHouse/ClickHouse/issues/10998). [#19750](https://github.com/ClickHouse/ClickHouse/pull/19750) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Clickhouse client query param CTE added test. [#19762](https://github.com/ClickHouse/ClickHouse/pull/19762) ([Maksim Kita](https://github.com/kitaisreal)). +* Correctly output infinite arguments for `formatReadableTimeDelta` function. In previous versions, there was implicit conversion to implementation specific integer value. [#19791](https://github.com/ClickHouse/ClickHouse/pull/19791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* `S3` table function now supports `auto` compression mode (autodetect). This closes [#18754](https://github.com/ClickHouse/ClickHouse/issues/18754). [#19793](https://github.com/ClickHouse/ClickHouse/pull/19793) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Set charset to utf8mb4 when interacting with remote MySQL servers. Fixes [#19795](https://github.com/ClickHouse/ClickHouse/issues/19795). [#19800](https://github.com/ClickHouse/ClickHouse/pull/19800) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `--reconnect` option to `clickhouse-benchmark`. When this option is specified, it will reconnect before every request. This is needed for testing. [#19872](https://github.com/ClickHouse/ClickHouse/pull/19872) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* fix data type convert issue for mysql engine ... [#18124](https://github.com/ClickHouse/ClickHouse/pull/18124) ([bo zeng](https://github.com/mis98zb)). +* `SELECT count() FROM table` now can be executed if only one any column can be selected from the `table`. This PR fixes [#10639](https://github.com/ClickHouse/ClickHouse/issues/10639). [#18233](https://github.com/ClickHouse/ClickHouse/pull/18233) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix index analysis of binary functions with constant argument which leads to wrong query results. This fixes [#18364](https://github.com/ClickHouse/ClickHouse/issues/18364). [#18373](https://github.com/ClickHouse/ClickHouse/pull/18373) ([Amos Bird](https://github.com/amosbird)). +* Disable constant folding for subqueries on the analysis stage, when the result cannot be calculated. [#18446](https://github.com/ClickHouse/ClickHouse/pull/18446) ([Azat Khuzhin](https://github.com/azat)). +* Attach partition should reset the mutation. [#18804](https://github.com/ClickHouse/ClickHouse/issues/18804). [#18935](https://github.com/ClickHouse/ClickHouse/pull/18935) ([fastio](https://github.com/fastio)). +* Fix bug when mutation with some escaped text (like `ALTER ... UPDATE e = CAST('foo', 'Enum8(\'foo\' = 1')` serialized incorrectly. Fixes [#18878](https://github.com/ClickHouse/ClickHouse/issues/18878). [#18944](https://github.com/ClickHouse/ClickHouse/pull/18944) ([alesapin](https://github.com/alesapin)). +* Fix error `Task was not found in task queue` (possible only for remote queries, with `async_socket_for_remote = 1`). [#18964](https://github.com/ClickHouse/ClickHouse/pull/18964) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* to fix [#18894](https://github.com/ClickHouse/ClickHouse/issues/18894) Add a check to avoid exception when long column alias('table.column' style, usually auto-generated by BI tools like Looker) equals to long table name. [#18968](https://github.com/ClickHouse/ClickHouse/pull/18968) ([Daniel Qin](https://github.com/mathfool)). +* Fix incorrect behavior when `ALTER TABLE ... DROP PART 'part_name'` query removes all deduplication blocks for the whole partition. Fixes [#18874](https://github.com/ClickHouse/ClickHouse/issues/18874). [#18969](https://github.com/ClickHouse/ClickHouse/pull/18969) ([alesapin](https://github.com/alesapin)). +* Fixed rare crashes when server run out of memory. [#18976](https://github.com/ClickHouse/ClickHouse/pull/18976) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed very rare deadlock at shutdown. [#18977](https://github.com/ClickHouse/ClickHouse/pull/18977) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix possible exception `QueryPipeline stream: different number of columns` caused by merging of query plan's `Expression` steps. Fixes [#18190](https://github.com/ClickHouse/ClickHouse/issues/18190). [#18980](https://github.com/ClickHouse/ClickHouse/pull/18980) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Disable `optimize_move_functions_out_of_any` because optimization is not always correct. This closes [#18051](https://github.com/ClickHouse/ClickHouse/issues/18051). This closes [#18973](https://github.com/ClickHouse/ClickHouse/issues/18973). [#18981](https://github.com/ClickHouse/ClickHouse/pull/18981) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Join tries to materialize const columns, but our code waits for them in other places. [#18982](https://github.com/ClickHouse/ClickHouse/pull/18982) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix inserting of `LowCardinality` column to table with `TinyLog` engine. Fixes [#18629](https://github.com/ClickHouse/ClickHouse/issues/18629). [#19010](https://github.com/ClickHouse/ClickHouse/pull/19010) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible error `Expected single dictionary argument for function` if use function `ignore` with `LowCardinality` argument. Fixes [#14275](https://github.com/ClickHouse/ClickHouse/issues/14275). [#19016](https://github.com/ClickHouse/ClickHouse/pull/19016) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Make sure `groupUniqArray` returns correct type for argument of Enum type. This closes [#17875](https://github.com/ClickHouse/ClickHouse/issues/17875). [#19019](https://github.com/ClickHouse/ClickHouse/pull/19019) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Restrict `MODIFY TTL` queries for `MergeTree` tables created in old syntax. Previously the query succeeded, but actually it had no effect. [#19064](https://github.com/ClickHouse/ClickHouse/pull/19064) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed `There is no checkpoint` error when inserting data through http interface using `Template` or `CustomSeparated` format. Fixes [#19021](https://github.com/ClickHouse/ClickHouse/issues/19021). [#19072](https://github.com/ClickHouse/ClickHouse/pull/19072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Simplify the implementation of `tupleHammingDistance`. Support for tuples of any equal length. Fixes [#19029](https://github.com/ClickHouse/ClickHouse/issues/19029). [#19084](https://github.com/ClickHouse/ClickHouse/pull/19084) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix startup bug when clickhouse was not able to read compression codec from `LowCardinality(Nullable(...))` and throws exception `Attempt to read after EOF`. Fixes [#18340](https://github.com/ClickHouse/ClickHouse/issues/18340). [#19101](https://github.com/ClickHouse/ClickHouse/pull/19101) ([alesapin](https://github.com/alesapin)). +* Fix bug in merge tree data writer which can lead to marks with bigger size than fixed granularity size. Fixes [#18913](https://github.com/ClickHouse/ClickHouse/issues/18913). [#19123](https://github.com/ClickHouse/ClickHouse/pull/19123) ([alesapin](https://github.com/alesapin)). +* Fix infinite reading from file in `ORC` format (was introduced in [#10580](https://github.com/ClickHouse/ClickHouse/issues/10580)). Fixes [#19095](https://github.com/ClickHouse/ClickHouse/issues/19095). [#19134](https://github.com/ClickHouse/ClickHouse/pull/19134) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* - Split RemoteQueryExecutorReadContext into module part - Fix leaking of pipe fd for `async_socket_for_remote`. [#19153](https://github.com/ClickHouse/ClickHouse/pull/19153) ([Azat Khuzhin](https://github.com/azat)). +* Fix bug when concurrent `ALTER` and `DROP` queries may hang while processing ReplicatedMergeTree table. [#19237](https://github.com/ClickHouse/ClickHouse/pull/19237) ([alesapin](https://github.com/alesapin)). +* Do not mark file for distributed send as broken on EOF. [#19290](https://github.com/ClickHouse/ClickHouse/pull/19290) ([Azat Khuzhin](https://github.com/azat)). +* Fix error `Cannot convert column now64() because it is constant but values of constants are different in source and result`. Continuation of [#7156](https://github.com/ClickHouse/ClickHouse/issues/7156). [#19316](https://github.com/ClickHouse/ClickHouse/pull/19316) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed possible wrong result or segfault on aggregation when Materialized View and its target table have different structure. Fixes [#18063](https://github.com/ClickHouse/ClickHouse/issues/18063). [#19322](https://github.com/ClickHouse/ClickHouse/pull/19322) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix system.parts _state column (LOGICAL_ERROR when querying this column, due to incorrect order). [#19346](https://github.com/ClickHouse/ClickHouse/pull/19346) ([Azat Khuzhin](https://github.com/azat)). +* Added `cast`, `accurateCast`, `accurateCastOrNull` performance tests. [#19354](https://github.com/ClickHouse/ClickHouse/pull/19354) ([Maksim Kita](https://github.com/kitaisreal)). +* - Fix default value in join types with non-zero default (e.g. some Enums). Closes [#18197](https://github.com/ClickHouse/ClickHouse/issues/18197). [#19360](https://github.com/ClickHouse/ClickHouse/pull/19360) ([Vladimir C](https://github.com/vdimir)). +* Fix possible buffer overflow in Uber H3 library. See https://github.com/uber/h3/issues/392. This closes [#19219](https://github.com/ClickHouse/ClickHouse/issues/19219). [#19383](https://github.com/ClickHouse/ClickHouse/pull/19383) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Uninitialized memory read was possible in encrypt/decrypt functions if empty string was passed as IV. This closes [#19391](https://github.com/ClickHouse/ClickHouse/issues/19391). [#19397](https://github.com/ClickHouse/ClickHouse/pull/19397) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix possible error `Extremes transform was already added to pipeline`. Fixes [#14100](https://github.com/ClickHouse/ClickHouse/issues/14100). [#19430](https://github.com/ClickHouse/ClickHouse/pull/19430) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed very rare bug that might cause mutation to hang after `DROP/DETACH/REPLACE/MOVE PARTITION`. It was partially fixed by [#15537](https://github.com/ClickHouse/ClickHouse/issues/15537) for the most cases. [#19443](https://github.com/ClickHouse/ClickHouse/pull/19443) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Mark distributed batch as broken in case of empty data block in one of files. [#19449](https://github.com/ClickHouse/ClickHouse/pull/19449) ([Azat Khuzhin](https://github.com/azat)). +* Buffer overflow (on memory read) was possible if `addMonth` function was called with specifically crafted arguments. This fixes [#19441](https://github.com/ClickHouse/ClickHouse/issues/19441). This fixes [#19413](https://github.com/ClickHouse/ClickHouse/issues/19413). [#19472](https://github.com/ClickHouse/ClickHouse/pull/19472) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong deserialization of columns description. It makes INSERT into a table with a column named `\` impossible. [#19479](https://github.com/ClickHouse/ClickHouse/pull/19479) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix SIGSEGV with merge_tree_min_rows_for_concurrent_read/merge_tree_min_bytes_for_concurrent_read=0/UINT64_MAX. [#19528](https://github.com/ClickHouse/ClickHouse/pull/19528) ([Azat Khuzhin](https://github.com/azat)). +* Query CREATE DICTIONARY id expression fix. [#19571](https://github.com/ClickHouse/ClickHouse/pull/19571) ([Maksim Kita](https://github.com/kitaisreal)). +* `DROP/DETACH TABLE table ON CLUSTER cluster SYNC` query might hang, it's fixed. Fixes [#19568](https://github.com/ClickHouse/ClickHouse/issues/19568). [#19572](https://github.com/ClickHouse/ClickHouse/pull/19572) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix use-after-free of the CompressedWriteBuffer in Connection after disconnect. [#19599](https://github.com/ClickHouse/ClickHouse/pull/19599) ([Azat Khuzhin](https://github.com/azat)). +* Fix wrong result of function `neighbor` for `LowCardinality` argument. Fixes [#10333](https://github.com/ClickHouse/ClickHouse/issues/10333). [#19617](https://github.com/ClickHouse/ClickHouse/pull/19617) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Some functions with big integers may cause segfault. Big integers is experimental feature. This closes [#19667](https://github.com/ClickHouse/ClickHouse/issues/19667). [#19672](https://github.com/ClickHouse/ClickHouse/pull/19672) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix rare bug when some replicated operations (like mutation) cannot process some parts after data corruption. Fixes [#19593](https://github.com/ClickHouse/ClickHouse/issues/19593). [#19702](https://github.com/ClickHouse/ClickHouse/pull/19702) ([alesapin](https://github.com/alesapin)). +* Fix a segmentation fault in `bitmapAndnot` function. Fixes [#19668](https://github.com/ClickHouse/ClickHouse/issues/19668). [#19713](https://github.com/ClickHouse/ClickHouse/pull/19713) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix crash when nested column name was used in `WHERE` or `PREWHERE`. Fixes [#19755](https://github.com/ClickHouse/ClickHouse/issues/19755). [#19763](https://github.com/ClickHouse/ClickHouse/pull/19763) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed stack overflow when using accurate comparison of arithmetic type with string type. [#19773](https://github.com/ClickHouse/ClickHouse/pull/19773) ([Alexander Tokmakov](https://github.com/tavplubix)). +* In previous versions, unusual arguments for function arrayEnumerateUniq may cause crash or infinite loop. This closes [#19787](https://github.com/ClickHouse/ClickHouse/issues/19787). [#19788](https://github.com/ClickHouse/ClickHouse/pull/19788) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The function `greatCircleAngle` returned inaccurate results in previous versions. This closes [#19769](https://github.com/ClickHouse/ClickHouse/issues/19769). [#19789](https://github.com/ClickHouse/ClickHouse/pull/19789) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix filtering by UInt8 greater than 127. [#19799](https://github.com/ClickHouse/ClickHouse/pull/19799) ([Anton Popov](https://github.com/CurtizJ)). +* Fix crash when pushing down predicates to union distinct subquery. This fixes [#19855](https://github.com/ClickHouse/ClickHouse/issues/19855). [#19861](https://github.com/ClickHouse/ClickHouse/pull/19861) ([Amos Bird](https://github.com/amosbird)). +* Fix argMin/argMax crash when combining with -If. This fixes https://clickhouse-test-reports.s3.yandex.net/19800/7b8589dbde5bc621d1bcfd68a713e4684183f593/fuzzer_ubsan/report.html#fail1. [#19868](https://github.com/ClickHouse/ClickHouse/pull/19868) ([Amos Bird](https://github.com/amosbird)). + +#### Build/Testing/Packaging Improvement +* Restore Kafka input in FreeBSD builds. [#18924](https://github.com/ClickHouse/ClickHouse/pull/18924) ([Alexandre Snarskii](https://github.com/snar)). +* Add integration tests run with memory sanitizer. [#18974](https://github.com/ClickHouse/ClickHouse/pull/18974) ([alesapin](https://github.com/alesapin)). +* Add SQLancer test docker image to run check in CI. [#19006](https://github.com/ClickHouse/ClickHouse/pull/19006) ([Ilya Yatsishin](https://github.com/qoega)). +* Added tests for defaults in URL and File engine. This closes [#5666](https://github.com/ClickHouse/ClickHouse/issues/5666). [#19015](https://github.com/ClickHouse/ClickHouse/pull/19015) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* - Enabling RBAC tests - Tests for SYSTEM privileges - Requirements name changes. [#19017](https://github.com/ClickHouse/ClickHouse/pull/19017) ([MyroTk](https://github.com/MyroTk)). +* Query Fuzzer will fuzz newly added tests more extensively. This closes [#18916](https://github.com/ClickHouse/ClickHouse/issues/18916). [#19185](https://github.com/ClickHouse/ClickHouse/pull/19185) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow building librdkafka without ssl. [#19337](https://github.com/ClickHouse/ClickHouse/pull/19337) ([filimonov](https://github.com/filimonov)). +* Avoid UBSan reports in `arrayElement` function, `substring` and `arraySum`. Fixes [#19305](https://github.com/ClickHouse/ClickHouse/issues/19305). Fixes [#19287](https://github.com/ClickHouse/ClickHouse/issues/19287). This closes [#19336](https://github.com/ClickHouse/ClickHouse/issues/19336). [#19347](https://github.com/ClickHouse/ClickHouse/pull/19347) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potential nullptr dereference in table function `VALUES`. [#19357](https://github.com/ClickHouse/ClickHouse/pull/19357) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow building ClickHouse with Kafka support on arm64. [#19369](https://github.com/ClickHouse/ClickHouse/pull/19369) ([filimonov](https://github.com/filimonov)). +* Integrate with [Big List of Naughty Strings](https://github.com/minimaxir/big-list-of-naughty-strings/) for better fuzzing. [#19480](https://github.com/ClickHouse/ClickHouse/pull/19480) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to explicitly enable or disable watchdog via environment variable `CLICKHOUSE_WATCHDOG_ENABLE`. By default it is enabled if server is not attached to terminal. [#19522](https://github.com/ClickHouse/ClickHouse/pull/19522) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Updating TestFlows AES encryption tests to support changes to the encrypt plaintext parameter. [#19674](https://github.com/ClickHouse/ClickHouse/pull/19674) ([vzakaznikov](https://github.com/vzakaznikov)). +* Made generation of macros.xml easier for integration tests. No more excessive logging from dicttoxml. dicttoxml project is not active for 5+ years. [#19697](https://github.com/ClickHouse/ClickHouse/pull/19697) ([Ilya Yatsishin](https://github.com/qoega)). +* Remove --project-directory for docker-compose in integration test. Fix logs formatting from docker container. [#19706](https://github.com/ClickHouse/ClickHouse/pull/19706) ([Ilya Yatsishin](https://github.com/qoega)). +* Fixed MemorySanitizer errors in cyrus-sasl and musl. [#19821](https://github.com/ClickHouse/ClickHouse/pull/19821) ([Ilya Yatsishin](https://github.com/qoega)). +* Add test for throwing an exception on inserting incorrect data in CollapsingMergeTree. [#19851](https://github.com/ClickHouse/ClickHouse/pull/19851) ([Kruglov Pavel](https://github.com/Avogar)). +* Adding retries for docker-compose start, stop and restart in TestFlows tests. [#19852](https://github.com/ClickHouse/ClickHouse/pull/19852) ([vzakaznikov](https://github.com/vzakaznikov)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Remove useless codes'. [#19293](https://github.com/ClickHouse/ClickHouse/pull/19293) ([sundyli](https://github.com/sundy-li)). +* NO CL ENTRY: 'Merging [#19387](https://github.com/ClickHouse/ClickHouse/issues/19387)'. [#19683](https://github.com/ClickHouse/ClickHouse/pull/19683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.2.10.48-stable.md b/docs/changelogs/v21.2.10.48-stable.md new file mode 100644 index 00000000000..11eea960931 --- /dev/null +++ b/docs/changelogs/v21.2.10.48-stable.md @@ -0,0 +1,11 @@ +### ClickHouse release v21.2.10.48-stable FIXME as compared to v21.2.9.41-stable + +#### Improvement +* Backported in [#23015](https://github.com/ClickHouse/ClickHouse/issues/23015): Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23080](https://github.com/ClickHouse/ClickHouse/issues/23080): Raised the threshold on max number of matches in result of the function `extractAllGroupsHorizontal`. [#23036](https://github.com/ClickHouse/ClickHouse/pull/23036) ([Vasily Nemkov](https://github.com/Enmk)). + +#### Bug Fix +* Backported in [#23155](https://github.com/ClickHouse/ClickHouse/issues/23155): Fixed a bug with unlimited wait for auxiliary AWS requests. [#22594](https://github.com/ClickHouse/ClickHouse/pull/22594) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#23032](https://github.com/ClickHouse/ClickHouse/issues/23032): Fix error `Cannot find column in ActionsDAG result` which may happen if subquery uses `untuple`. Fixes [#22290](https://github.com/ClickHouse/ClickHouse/issues/22290). [#22991](https://github.com/ClickHouse/ClickHouse/pull/22991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#23171](https://github.com/ClickHouse/ClickHouse/issues/23171): Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.2.2.8-stable.md b/docs/changelogs/v21.2.2.8-stable.md new file mode 100644 index 00000000000..368243120f1 --- /dev/null +++ b/docs/changelogs/v21.2.2.8-stable.md @@ -0,0 +1,166 @@ +### ClickHouse release v21.2.2.8-stable FIXME as compared to v21.1.1.5646-prestable + +#### Backward Incompatible Change +* - Fix memory tracking for `OPTIMIZE TABLE`/merges - Account query memory limits and sampling for `OPTIMIZE TABLE`/merges. [#18772](https://github.com/ClickHouse/ClickHouse/pull/18772) ([Azat Khuzhin](https://github.com/azat)). +* Forbid `lcm`/`gcd` for floats. [#19532](https://github.com/ClickHouse/ClickHouse/pull/19532) ([Azat Khuzhin](https://github.com/azat)). +* Bitwise functions (`bitAnd`, `bitOr`, etc) are forbidden for floating point arguments. Now you have to do explicit cast to integer. [#19853](https://github.com/ClickHouse/ClickHouse/pull/19853) ([Azat Khuzhin](https://github.com/azat)). + +#### New Feature +* add support for zstd long option for better compression of string columns to save space. [#17184](https://github.com/ClickHouse/ClickHouse/pull/17184) ([ygrek](https://github.com/ygrek)). +* - Added support of mapping LDAP group names, and attribute values in general, to local roles for users from ldap user directories. [#17211](https://github.com/ClickHouse/ClickHouse/pull/17211) ([Denis Glazachev](https://github.com/traceon)). +* Data type `Nested` now supports arbitrary levels of nesting. Introduced subcolumns of complex types, such as `size0` in `Array`, `null` in `Nullable`, names of `Tuple` elements, which can be read without reading of whole column. [#17310](https://github.com/ClickHouse/ClickHouse/pull/17310) ([Anton Popov](https://github.com/CurtizJ)). +* Add support of tuple argument to `argMin` and `argMax` functions. [#17359](https://github.com/ClickHouse/ClickHouse/pull/17359) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Added `Nullable` support for `FlatDictionary`, `HashedDictionary`, `ComplexKeyHashedDictionary`, `DirectDictionary`, `ComplexKeyDirectDictionary`, `RangeHashedDictionary`. [#18236](https://github.com/ClickHouse/ClickHouse/pull/18236) ([Maksim Kita](https://github.com/kitaisreal)). +* Disallow floating point column as partition key related to : [#18421](https://github.com/ClickHouse/ClickHouse/issues/18421)#event-4147046255. [#18464](https://github.com/ClickHouse/ClickHouse/pull/18464) ([hexiaoting](https://github.com/hexiaoting)). +* Add function decodeXMLComponent to decode characters for XML. ``` SELECT decodeXMLComponent('Hello,"world"!'); ``` [#17659](https://github.com/ClickHouse/ClickHouse/issues/17659). [#18542](https://github.com/ClickHouse/ClickHouse/pull/18542) ([nauta](https://github.com/nautaa)). +* Added PostgreSQL table engine (both select/insert, with support for multidimensional arrays), also as table function. Added PostgreSQL dictionary source. Added PostgreSQL database engine. [#18554](https://github.com/ClickHouse/ClickHouse/pull/18554) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add `SELECT ALL` syntax. closes [#18706](https://github.com/ClickHouse/ClickHouse/issues/18706). [#18723](https://github.com/ClickHouse/ClickHouse/pull/18723) ([flynn](https://github.com/ucasfl)). +* Add three functions for map data type: 1. mapContains(map, key) to check weather map.keys include the second parameter key. 2. mapKeys(map) return all the keys in Array format 3. mapValues(map) return all the values in Array format. [#18788](https://github.com/ClickHouse/ClickHouse/pull/18788) ([hexiaoting](https://github.com/hexiaoting)). +* Support MetaKey+Enter hotkey binding in play ui. [#19012](https://github.com/ClickHouse/ClickHouse/pull/19012) ([sundyli](https://github.com/sundy-li)). +* Function formatDateTime support the %Q modification to format date to quarter. ... [#19224](https://github.com/ClickHouse/ClickHouse/pull/19224) ([Jianmei Zhang](https://github.com/zhangjmruc)). +* ... [#19261](https://github.com/ClickHouse/ClickHouse/pull/19261) ([RegulusZ](https://github.com/RegulusZ)). +* Add factories' objects names, created during query, into system.query_log. Closes [#18495](https://github.com/ClickHouse/ClickHouse/issues/18495). [#19371](https://github.com/ClickHouse/ClickHouse/pull/19371) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add `sign` math function. [#19527](https://github.com/ClickHouse/ClickHouse/pull/19527) ([flynn](https://github.com/ucasfl)). +* Added functions `parseDateTimeBestEffortUSOrZero`, `parseDateTimeBestEffortUSOrNull`. [#19712](https://github.com/ClickHouse/ClickHouse/pull/19712) ([Maksim Kita](https://github.com/kitaisreal)). +* ... [#19764](https://github.com/ClickHouse/ClickHouse/pull/19764) ([emhlbmc](https://github.com/emhlbmc)). + +#### Performance Improvement +* Use a connection pool for S3 connections, controlled by the `s3_max_connections` settings. [#13405](https://github.com/ClickHouse/ClickHouse/pull/13405) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Rewrite `sumIf()` and `sum(if())` function to `countIf()` function when logically equivalent. [#17041](https://github.com/ClickHouse/ClickHouse/pull/17041) ([flynn](https://github.com/ucasfl)). +* Update libcxx and use unstable ABI to provide better performance. [#18914](https://github.com/ClickHouse/ClickHouse/pull/18914) ([Daniel Kutenin](https://github.com/danlark1)). +* Faster parts removal by lowering the number of `stat` syscalls. This returns the optimization that existed while ago. More safe interface of `IDisk`. This closes [#19065](https://github.com/ClickHouse/ClickHouse/issues/19065). [#19086](https://github.com/ClickHouse/ClickHouse/pull/19086) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Speed up aggregate function `sum`. Improvement only visible on synthetic benchmarks and not very practical. [#19216](https://github.com/ClickHouse/ClickHouse/pull/19216) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support splitting `Filter` step of query plan into `Expression + Filter` pair. Together with `Expression + Expression` merging optimization ([#17458](https://github.com/ClickHouse/ClickHouse/issues/17458)) it may delay execution for some expressions after `Filter` step. [#19253](https://github.com/ClickHouse/ClickHouse/pull/19253) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reduce lock contention for multiple layers of the Buffer engine. [#19379](https://github.com/ClickHouse/ClickHouse/pull/19379) ([Azat Khuzhin](https://github.com/azat)). +* Slightly improve server latency by removing access to configuration on every connection. [#19863](https://github.com/ClickHouse/ClickHouse/pull/19863) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Added support for `WITH ... [AND] [PERIODIC] REFRESH [interval_in_sec]` clause when creating `LIVE VIEW` tables. [#14822](https://github.com/ClickHouse/ClickHouse/pull/14822) ([vzakaznikov](https://github.com/vzakaznikov)). +* - Add optimize_alias_column_prediction (on by default), that will: * Respect aliased columns in WHERE during partition pruning and skipping data using secondary indexes * Respect aliased columns in WHERE for trivial count queries for optimize_trivial_count * Respect aliased columns in GROUP BY/ORDER BY for optimize_aggregation_in_order/optimize_read_in_order. [#16995](https://github.com/ClickHouse/ClickHouse/pull/16995) ([sundyli](https://github.com/sundy-li)). +* Updated AWS C++ SDK in order to utilize global regions in S3. [#17870](https://github.com/ClickHouse/ClickHouse/pull/17870) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Support insert into table function `cluster`, and for both table functions `remote` and `cluster`, support distributing data across nodes by specify sharding key. Close [#16752](https://github.com/ClickHouse/ClickHouse/issues/16752). [#18264](https://github.com/ClickHouse/ClickHouse/pull/18264) ([flynn](https://github.com/ucasfl)). +* Support `EXISTS VIEW` syntax. [#18552](https://github.com/ClickHouse/ClickHouse/pull/18552) ([Du Chuan](https://github.com/spongedu)). +* Update librdkafka to v1.6.0-RC2. Fixes [#18668](https://github.com/ClickHouse/ClickHouse/issues/18668). [#18671](https://github.com/ClickHouse/ClickHouse/pull/18671) ([filimonov](https://github.com/filimonov)). +* Allow CTE to be further aliased. Propagate CSE to subqueries in the same level when `enable_global_with_statement = 1`. This fixes [#17378](https://github.com/ClickHouse/ClickHouse/issues/17378) . This fixes https://github.com/ClickHouse/ClickHouse/pull/16575#issuecomment-753416235 . [#18684](https://github.com/ClickHouse/ClickHouse/pull/18684) ([Amos Bird](https://github.com/amosbird)). +* Add [UInt8, UInt16, UInt32, UInt64] arguments types support for bitmapTransform, bitmapSubsetInRange, bitmapSubsetLimit, bitmapContains functions. This closes [#18713](https://github.com/ClickHouse/ClickHouse/issues/18713). [#18791](https://github.com/ClickHouse/ClickHouse/pull/18791) ([sundyli](https://github.com/sundy-li)). +* Added prefix-based S3 endpoint settings. [#18812](https://github.com/ClickHouse/ClickHouse/pull/18812) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix issues with RIGHT and FULL JOIN of tables with aggregate function states. In previous versions exception about `cloneResized` method was thrown. [#18818](https://github.com/ClickHouse/ClickHouse/pull/18818) ([templarzq](https://github.com/templarzq)). +* - Check per-block checksum of the distributed batch on the sender before sending (without reading the file twice, the checksums will be verified while reading), this will avoid stuck of the INSERT on the receiver (on truncated .bin file on the sender) - Avoid reading .bin files twice for batched INSERT (it was required to calculate rows/bytes to take squashing into account, now this information included into the header, backward compatible is preserved). [#18853](https://github.com/ClickHouse/ClickHouse/pull/18853) ([Azat Khuzhin](https://github.com/azat)). +* Add `normalizeQueryKeepNames` and `normalizedQueryHashKeepNames` to normalize queries without masking long names with `?`. This helps better analyze complex query logs. [#18910](https://github.com/ClickHouse/ClickHouse/pull/18910) ([Amos Bird](https://github.com/amosbird)). +* Docker image: several improvements for clickhouse-server entrypoint. [#18954](https://github.com/ClickHouse/ClickHouse/pull/18954) ([filimonov](https://github.com/filimonov)). +* Fixed `PeekableReadBuffer: Memory limit exceed` error when inserting data with huge strings. Fixes [#18690](https://github.com/ClickHouse/ClickHouse/issues/18690). [#18979](https://github.com/ClickHouse/ClickHouse/pull/18979) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Explicitly set uid / gid of clickhouse user & group to the fixed values (101) in clickhouse-server images. [#19096](https://github.com/ClickHouse/ClickHouse/pull/19096) ([filimonov](https://github.com/filimonov)). +* The exception when function `bar` is called with certain NaN argument may be slightly misleading in previous versions. This fixes [#19088](https://github.com/ClickHouse/ClickHouse/issues/19088). [#19107](https://github.com/ClickHouse/ClickHouse/pull/19107) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow change `max_server_memory_usage` without restart. This closes [#18154](https://github.com/ClickHouse/ClickHouse/issues/18154). [#19186](https://github.com/ClickHouse/ClickHouse/pull/19186) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong alignment of values of `IPv4` data type in Pretty formats. They were aligned to the right, not to the left. This closes [#19184](https://github.com/ClickHouse/ClickHouse/issues/19184). [#19339](https://github.com/ClickHouse/ClickHouse/pull/19339) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow docker to be executed with arbitrary uid. [#19374](https://github.com/ClickHouse/ClickHouse/pull/19374) ([filimonov](https://github.com/filimonov)). +* Add metrics for MergeTree parts (Wide/Compact/InMemory) types. [#19381](https://github.com/ClickHouse/ClickHouse/pull/19381) ([Azat Khuzhin](https://github.com/azat)). +* Improve MySQL compatibility. [#19387](https://github.com/ClickHouse/ClickHouse/pull/19387) ([Daniil Kondratyev](https://github.com/dankondr)). +* Add `http_referer` field to `system.query_log`, `system.processes`, etc. This closes [#19389](https://github.com/ClickHouse/ClickHouse/issues/19389). [#19390](https://github.com/ClickHouse/ClickHouse/pull/19390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* `toIPv6` function parses `IPv4` addresses. [#19518](https://github.com/ClickHouse/ClickHouse/pull/19518) ([Bharat Nallan](https://github.com/bharatnc)). +* Support using the new location of `.debug` file. This fixes [#19348](https://github.com/ClickHouse/ClickHouse/issues/19348). [#19520](https://github.com/ClickHouse/ClickHouse/pull/19520) ([Amos Bird](https://github.com/amosbird)). +* Enable function length/empty/notEmpty for datatype map, which returns keys number in map. [#19530](https://github.com/ClickHouse/ClickHouse/pull/19530) ([李扬](https://github.com/taiyang-li)). +* Support constant result in function `multiIf`. [#19533](https://github.com/ClickHouse/ClickHouse/pull/19533) ([Maksim Kita](https://github.com/kitaisreal)). +* Add an option to disable validation of checksums on reading. Should never be used in production. Please do not expect any benefits in disabling it. It may only be used for experiments and benchmarks. The setting only applicable for tables of MergeTree family. Checksums are always validated for other table engines and when receiving data over network. In my observations there is no performance difference or it is less than 0.5%. [#19588](https://github.com/ClickHouse/ClickHouse/pull/19588) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dictionary better error message during attribute parsing. [#19678](https://github.com/ClickHouse/ClickHouse/pull/19678) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix rare `max_number_of_merges_with_ttl_in_pool` limit overrun (more merges with TTL can be assigned) for non-replicated MergeTree. [#19708](https://github.com/ClickHouse/ClickHouse/pull/19708) ([alesapin](https://github.com/alesapin)). +* Insuffiient arguments check in `positionCaseInsensitiveUTF8` function triggered address sanitizer. [#19720](https://github.com/ClickHouse/ClickHouse/pull/19720) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add separate pool for message brokers (RabbitMQ and Kafka). [#19722](https://github.com/ClickHouse/ClickHouse/pull/19722) ([Azat Khuzhin](https://github.com/azat)). +* In distributed queries if the setting `async_socket_for_remote` is enabled, it was possible to get stack overflow at least in debug build configuration if very deeply nested data type is used in table (e.g. `Array(Array(Array(...more...)))`). This fixes [#19108](https://github.com/ClickHouse/ClickHouse/issues/19108). This change introduces minor backward incompatibility: excessive parenthesis in type definitions no longer supported, example: `Array((UInt8))`. [#19736](https://github.com/ClickHouse/ClickHouse/pull/19736) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Table function `S3` will use global region if the region can't be determined exactly. This closes [#10998](https://github.com/ClickHouse/ClickHouse/issues/10998). [#19750](https://github.com/ClickHouse/ClickHouse/pull/19750) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Clickhouse client query param CTE added test. [#19762](https://github.com/ClickHouse/ClickHouse/pull/19762) ([Maksim Kita](https://github.com/kitaisreal)). +* Correctly output infinite arguments for `formatReadableTimeDelta` function. In previous versions, there was implicit conversion to implementation specific integer value. [#19791](https://github.com/ClickHouse/ClickHouse/pull/19791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* `S3` table function now supports `auto` compression mode (autodetect). This closes [#18754](https://github.com/ClickHouse/ClickHouse/issues/18754). [#19793](https://github.com/ClickHouse/ClickHouse/pull/19793) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Set charset to utf8mb4 when interacting with remote MySQL servers. Fixes [#19795](https://github.com/ClickHouse/ClickHouse/issues/19795). [#19800](https://github.com/ClickHouse/ClickHouse/pull/19800) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `--reconnect` option to `clickhouse-benchmark`. When this option is specified, it will reconnect before every request. This is needed for testing. [#19872](https://github.com/ClickHouse/ClickHouse/pull/19872) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* fix data type convert issue for mysql engine ... [#18124](https://github.com/ClickHouse/ClickHouse/pull/18124) ([bo zeng](https://github.com/mis98zb)). +* `SELECT count() FROM table` now can be executed if only one any column can be selected from the `table`. This PR fixes [#10639](https://github.com/ClickHouse/ClickHouse/issues/10639). [#18233](https://github.com/ClickHouse/ClickHouse/pull/18233) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix index analysis of binary functions with constant argument which leads to wrong query results. This fixes [#18364](https://github.com/ClickHouse/ClickHouse/issues/18364). [#18373](https://github.com/ClickHouse/ClickHouse/pull/18373) ([Amos Bird](https://github.com/amosbird)). +* Disable constant folding for subqueries on the analysis stage, when the result cannot be calculated. [#18446](https://github.com/ClickHouse/ClickHouse/pull/18446) ([Azat Khuzhin](https://github.com/azat)). +* Attach partition should reset the mutation. [#18804](https://github.com/ClickHouse/ClickHouse/issues/18804). [#18935](https://github.com/ClickHouse/ClickHouse/pull/18935) ([fastio](https://github.com/fastio)). +* Fix bug when mutation with some escaped text (like `ALTER ... UPDATE e = CAST('foo', 'Enum8(\'foo\' = 1')` serialized incorrectly. Fixes [#18878](https://github.com/ClickHouse/ClickHouse/issues/18878). [#18944](https://github.com/ClickHouse/ClickHouse/pull/18944) ([alesapin](https://github.com/alesapin)). +* Fix error `Task was not found in task queue` (possible only for remote queries, with `async_socket_for_remote = 1`). [#18964](https://github.com/ClickHouse/ClickHouse/pull/18964) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* to fix [#18894](https://github.com/ClickHouse/ClickHouse/issues/18894) Add a check to avoid exception when long column alias('table.column' style, usually auto-generated by BI tools like Looker) equals to long table name. [#18968](https://github.com/ClickHouse/ClickHouse/pull/18968) ([Daniel Qin](https://github.com/mathfool)). +* Fix incorrect behavior when `ALTER TABLE ... DROP PART 'part_name'` query removes all deduplication blocks for the whole partition. Fixes [#18874](https://github.com/ClickHouse/ClickHouse/issues/18874). [#18969](https://github.com/ClickHouse/ClickHouse/pull/18969) ([alesapin](https://github.com/alesapin)). +* Fixed rare crashes when server run out of memory. [#18976](https://github.com/ClickHouse/ClickHouse/pull/18976) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed very rare deadlock at shutdown. [#18977](https://github.com/ClickHouse/ClickHouse/pull/18977) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix possible exception `QueryPipeline stream: different number of columns` caused by merging of query plan's `Expression` steps. Fixes [#18190](https://github.com/ClickHouse/ClickHouse/issues/18190). [#18980](https://github.com/ClickHouse/ClickHouse/pull/18980) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Disable `optimize_move_functions_out_of_any` because optimization is not always correct. This closes [#18051](https://github.com/ClickHouse/ClickHouse/issues/18051). This closes [#18973](https://github.com/ClickHouse/ClickHouse/issues/18973). [#18981](https://github.com/ClickHouse/ClickHouse/pull/18981) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Join tries to materialize const columns, but our code waits for them in other places. [#18982](https://github.com/ClickHouse/ClickHouse/pull/18982) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix inserting of `LowCardinality` column to table with `TinyLog` engine. Fixes [#18629](https://github.com/ClickHouse/ClickHouse/issues/18629). [#19010](https://github.com/ClickHouse/ClickHouse/pull/19010) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible error `Expected single dictionary argument for function` if use function `ignore` with `LowCardinality` argument. Fixes [#14275](https://github.com/ClickHouse/ClickHouse/issues/14275). [#19016](https://github.com/ClickHouse/ClickHouse/pull/19016) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Make sure `groupUniqArray` returns correct type for argument of Enum type. This closes [#17875](https://github.com/ClickHouse/ClickHouse/issues/17875). [#19019](https://github.com/ClickHouse/ClickHouse/pull/19019) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Restrict `MODIFY TTL` queries for `MergeTree` tables created in old syntax. Previously the query succeeded, but actually it had no effect. [#19064](https://github.com/ClickHouse/ClickHouse/pull/19064) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed `There is no checkpoint` error when inserting data through http interface using `Template` or `CustomSeparated` format. Fixes [#19021](https://github.com/ClickHouse/ClickHouse/issues/19021). [#19072](https://github.com/ClickHouse/ClickHouse/pull/19072) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Simplify the implementation of `tupleHammingDistance`. Support for tuples of any equal length. Fixes [#19029](https://github.com/ClickHouse/ClickHouse/issues/19029). [#19084](https://github.com/ClickHouse/ClickHouse/pull/19084) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix startup bug when clickhouse was not able to read compression codec from `LowCardinality(Nullable(...))` and throws exception `Attempt to read after EOF`. Fixes [#18340](https://github.com/ClickHouse/ClickHouse/issues/18340). [#19101](https://github.com/ClickHouse/ClickHouse/pull/19101) ([alesapin](https://github.com/alesapin)). +* Fix bug in merge tree data writer which can lead to marks with bigger size than fixed granularity size. Fixes [#18913](https://github.com/ClickHouse/ClickHouse/issues/18913). [#19123](https://github.com/ClickHouse/ClickHouse/pull/19123) ([alesapin](https://github.com/alesapin)). +* Fix infinite reading from file in `ORC` format (was introduced in [#10580](https://github.com/ClickHouse/ClickHouse/issues/10580)). Fixes [#19095](https://github.com/ClickHouse/ClickHouse/issues/19095). [#19134](https://github.com/ClickHouse/ClickHouse/pull/19134) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* - Split RemoteQueryExecutorReadContext into module part - Fix leaking of pipe fd for `async_socket_for_remote`. [#19153](https://github.com/ClickHouse/ClickHouse/pull/19153) ([Azat Khuzhin](https://github.com/azat)). +* Fix bug when concurrent `ALTER` and `DROP` queries may hang while processing ReplicatedMergeTree table. [#19237](https://github.com/ClickHouse/ClickHouse/pull/19237) ([alesapin](https://github.com/alesapin)). +* Do not mark file for distributed send as broken on EOF. [#19290](https://github.com/ClickHouse/ClickHouse/pull/19290) ([Azat Khuzhin](https://github.com/azat)). +* Fix error `Cannot convert column now64() because it is constant but values of constants are different in source and result`. Continuation of [#7156](https://github.com/ClickHouse/ClickHouse/issues/7156). [#19316](https://github.com/ClickHouse/ClickHouse/pull/19316) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed possible wrong result or segfault on aggregation when Materialized View and its target table have different structure. Fixes [#18063](https://github.com/ClickHouse/ClickHouse/issues/18063). [#19322](https://github.com/ClickHouse/ClickHouse/pull/19322) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix system.parts _state column (LOGICAL_ERROR when querying this column, due to incorrect order). [#19346](https://github.com/ClickHouse/ClickHouse/pull/19346) ([Azat Khuzhin](https://github.com/azat)). +* Added `cast`, `accurateCast`, `accurateCastOrNull` performance tests. [#19354](https://github.com/ClickHouse/ClickHouse/pull/19354) ([Maksim Kita](https://github.com/kitaisreal)). +* - Fix default value in join types with non-zero default (e.g. some Enums). Closes [#18197](https://github.com/ClickHouse/ClickHouse/issues/18197). [#19360](https://github.com/ClickHouse/ClickHouse/pull/19360) ([Vladimir C](https://github.com/vdimir)). +* Fix possible buffer overflow in Uber H3 library. See https://github.com/uber/h3/issues/392. This closes [#19219](https://github.com/ClickHouse/ClickHouse/issues/19219). [#19383](https://github.com/ClickHouse/ClickHouse/pull/19383) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Uninitialized memory read was possible in encrypt/decrypt functions if empty string was passed as IV. This closes [#19391](https://github.com/ClickHouse/ClickHouse/issues/19391). [#19397](https://github.com/ClickHouse/ClickHouse/pull/19397) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix possible error `Extremes transform was already added to pipeline`. Fixes [#14100](https://github.com/ClickHouse/ClickHouse/issues/14100). [#19430](https://github.com/ClickHouse/ClickHouse/pull/19430) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed very rare bug that might cause mutation to hang after `DROP/DETACH/REPLACE/MOVE PARTITION`. It was partially fixed by [#15537](https://github.com/ClickHouse/ClickHouse/issues/15537) for the most cases. [#19443](https://github.com/ClickHouse/ClickHouse/pull/19443) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Mark distributed batch as broken in case of empty data block in one of files. [#19449](https://github.com/ClickHouse/ClickHouse/pull/19449) ([Azat Khuzhin](https://github.com/azat)). +* Buffer overflow (on memory read) was possible if `addMonth` function was called with specifically crafted arguments. This fixes [#19441](https://github.com/ClickHouse/ClickHouse/issues/19441). This fixes [#19413](https://github.com/ClickHouse/ClickHouse/issues/19413). [#19472](https://github.com/ClickHouse/ClickHouse/pull/19472) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix wrong deserialization of columns description. It makes INSERT into a table with a column named `\` impossible. [#19479](https://github.com/ClickHouse/ClickHouse/pull/19479) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix SIGSEGV with merge_tree_min_rows_for_concurrent_read/merge_tree_min_bytes_for_concurrent_read=0/UINT64_MAX. [#19528](https://github.com/ClickHouse/ClickHouse/pull/19528) ([Azat Khuzhin](https://github.com/azat)). +* Query CREATE DICTIONARY id expression fix. [#19571](https://github.com/ClickHouse/ClickHouse/pull/19571) ([Maksim Kita](https://github.com/kitaisreal)). +* `DROP/DETACH TABLE table ON CLUSTER cluster SYNC` query might hang, it's fixed. Fixes [#19568](https://github.com/ClickHouse/ClickHouse/issues/19568). [#19572](https://github.com/ClickHouse/ClickHouse/pull/19572) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix use-after-free of the CompressedWriteBuffer in Connection after disconnect. [#19599](https://github.com/ClickHouse/ClickHouse/pull/19599) ([Azat Khuzhin](https://github.com/azat)). +* Fix wrong result of function `neighbor` for `LowCardinality` argument. Fixes [#10333](https://github.com/ClickHouse/ClickHouse/issues/10333). [#19617](https://github.com/ClickHouse/ClickHouse/pull/19617) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Some functions with big integers may cause segfault. Big integers is experimental feature. This closes [#19667](https://github.com/ClickHouse/ClickHouse/issues/19667). [#19672](https://github.com/ClickHouse/ClickHouse/pull/19672) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19986](https://github.com/ClickHouse/ClickHouse/issues/19986): Background thread which executes `ON CLUSTER` queries might hang waiting for dropped replicated table to do something. It's fixed. [#19684](https://github.com/ClickHouse/ClickHouse/pull/19684) ([yiguolei](https://github.com/yiguolei)). +* Fix rare bug when some replicated operations (like mutation) cannot process some parts after data corruption. Fixes [#19593](https://github.com/ClickHouse/ClickHouse/issues/19593). [#19702](https://github.com/ClickHouse/ClickHouse/pull/19702) ([alesapin](https://github.com/alesapin)). +* Fix a segmentation fault in `bitmapAndnot` function. Fixes [#19668](https://github.com/ClickHouse/ClickHouse/issues/19668). [#19713](https://github.com/ClickHouse/ClickHouse/pull/19713) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix crash when nested column name was used in `WHERE` or `PREWHERE`. Fixes [#19755](https://github.com/ClickHouse/ClickHouse/issues/19755). [#19763](https://github.com/ClickHouse/ClickHouse/pull/19763) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed stack overflow when using accurate comparison of arithmetic type with string type. [#19773](https://github.com/ClickHouse/ClickHouse/pull/19773) ([Alexander Tokmakov](https://github.com/tavplubix)). +* In previous versions, unusual arguments for function arrayEnumerateUniq may cause crash or infinite loop. This closes [#19787](https://github.com/ClickHouse/ClickHouse/issues/19787). [#19788](https://github.com/ClickHouse/ClickHouse/pull/19788) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The function `greatCircleAngle` returned inaccurate results in previous versions. This closes [#19769](https://github.com/ClickHouse/ClickHouse/issues/19769). [#19789](https://github.com/ClickHouse/ClickHouse/pull/19789) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19922](https://github.com/ClickHouse/ClickHouse/issues/19922): Fix clickhouse-client abort exception while executing only `select`. [#19790](https://github.com/ClickHouse/ClickHouse/pull/19790) ([李扬](https://github.com/taiyang-li)). +* Fix filtering by UInt8 greater than 127. [#19799](https://github.com/ClickHouse/ClickHouse/pull/19799) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#20007](https://github.com/ClickHouse/ClickHouse/issues/20007): Fix starting the server with tables having default expressions containing dictGet(). Allow getting return type of dictGet() without loading dictionary. [#19805](https://github.com/ClickHouse/ClickHouse/pull/19805) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix crash when pushing down predicates to union distinct subquery. This fixes [#19855](https://github.com/ClickHouse/ClickHouse/issues/19855). [#19861](https://github.com/ClickHouse/ClickHouse/pull/19861) ([Amos Bird](https://github.com/amosbird)). +* Fix argMin/argMax crash when combining with -If. This fixes https://clickhouse-test-reports.s3.yandex.net/19800/7b8589dbde5bc621d1bcfd68a713e4684183f593/fuzzer_ubsan/report.html#fail1. [#19868](https://github.com/ClickHouse/ClickHouse/pull/19868) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#19939](https://github.com/ClickHouse/ClickHouse/issues/19939): Deadlock was possible if system.text_log is enabled. This fixes [#19874](https://github.com/ClickHouse/ClickHouse/issues/19874). [#19875](https://github.com/ClickHouse/ClickHouse/pull/19875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#19935](https://github.com/ClickHouse/ClickHouse/issues/19935): BloomFilter index crash fix. Fixes [#19757](https://github.com/ClickHouse/ClickHouse/issues/19757). [#19884](https://github.com/ClickHouse/ClickHouse/pull/19884) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#19996](https://github.com/ClickHouse/ClickHouse/issues/19996): - Fix a segfault in function `fromModifiedJulianDay` when the argument type is `Nullable(T)` for any integral types other than Int32. [#19959](https://github.com/ClickHouse/ClickHouse/pull/19959) ([PHO](https://github.com/depressed-pho)). +* Backported in [#20112](https://github.com/ClickHouse/ClickHouse/issues/20112): `EmbeddedRocksDB` is an experimental storage. Fix the issue with lack of proper type checking. Simplified code. This closes [#19967](https://github.com/ClickHouse/ClickHouse/issues/19967). [#19972](https://github.com/ClickHouse/ClickHouse/pull/19972) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#20028](https://github.com/ClickHouse/ClickHouse/issues/20028): Prevent "Connection refused" in docker during initialization script execution. [#20012](https://github.com/ClickHouse/ClickHouse/pull/20012) ([filimonov](https://github.com/filimonov)). +* Backported in [#20123](https://github.com/ClickHouse/ClickHouse/issues/20123): MaterializeMySQL: Fix replication for statements that update several tables. [#20066](https://github.com/ClickHouse/ClickHouse/pull/20066) ([Håvard Kvålen](https://github.com/havardk)). +* Backported in [#20146](https://github.com/ClickHouse/ClickHouse/issues/20146): Fix server crash after query with `if` function with `Tuple` type of then/else branches result. `Tuple` type must contain `Array` or another complex type. Fixes [#18356](https://github.com/ClickHouse/ClickHouse/issues/18356). [#20133](https://github.com/ClickHouse/ClickHouse/pull/20133) ([alesapin](https://github.com/alesapin)). + +#### Build/Testing/Packaging Improvement +* Restore Kafka input in FreeBSD builds. [#18924](https://github.com/ClickHouse/ClickHouse/pull/18924) ([Alexandre Snarskii](https://github.com/snar)). +* Add integration tests run with memory sanitizer. [#18974](https://github.com/ClickHouse/ClickHouse/pull/18974) ([alesapin](https://github.com/alesapin)). +* Add SQLancer test docker image to run check in CI. [#19006](https://github.com/ClickHouse/ClickHouse/pull/19006) ([Ilya Yatsishin](https://github.com/qoega)). +* Added tests for defaults in URL and File engine. This closes [#5666](https://github.com/ClickHouse/ClickHouse/issues/5666). [#19015](https://github.com/ClickHouse/ClickHouse/pull/19015) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* - Enabling RBAC tests - Tests for SYSTEM privileges - Requirements name changes. [#19017](https://github.com/ClickHouse/ClickHouse/pull/19017) ([MyroTk](https://github.com/MyroTk)). +* Query Fuzzer will fuzz newly added tests more extensively. This closes [#18916](https://github.com/ClickHouse/ClickHouse/issues/18916). [#19185](https://github.com/ClickHouse/ClickHouse/pull/19185) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow building librdkafka without ssl. [#19337](https://github.com/ClickHouse/ClickHouse/pull/19337) ([filimonov](https://github.com/filimonov)). +* Avoid UBSan reports in `arrayElement` function, `substring` and `arraySum`. Fixes [#19305](https://github.com/ClickHouse/ClickHouse/issues/19305). Fixes [#19287](https://github.com/ClickHouse/ClickHouse/issues/19287). This closes [#19336](https://github.com/ClickHouse/ClickHouse/issues/19336). [#19347](https://github.com/ClickHouse/ClickHouse/pull/19347) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix potential nullptr dereference in table function `VALUES`. [#19357](https://github.com/ClickHouse/ClickHouse/pull/19357) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow building ClickHouse with Kafka support on arm64. [#19369](https://github.com/ClickHouse/ClickHouse/pull/19369) ([filimonov](https://github.com/filimonov)). +* Integrate with [Big List of Naughty Strings](https://github.com/minimaxir/big-list-of-naughty-strings/) for better fuzzing. [#19480](https://github.com/ClickHouse/ClickHouse/pull/19480) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to explicitly enable or disable watchdog via environment variable `CLICKHOUSE_WATCHDOG_ENABLE`. By default it is enabled if server is not attached to terminal. [#19522](https://github.com/ClickHouse/ClickHouse/pull/19522) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Updating TestFlows AES encryption tests to support changes to the encrypt plaintext parameter. [#19674](https://github.com/ClickHouse/ClickHouse/pull/19674) ([vzakaznikov](https://github.com/vzakaznikov)). +* Made generation of macros.xml easier for integration tests. No more excessive logging from dicttoxml. dicttoxml project is not active for 5+ years. [#19697](https://github.com/ClickHouse/ClickHouse/pull/19697) ([Ilya Yatsishin](https://github.com/qoega)). +* Remove --project-directory for docker-compose in integration test. Fix logs formatting from docker container. [#19706](https://github.com/ClickHouse/ClickHouse/pull/19706) ([Ilya Yatsishin](https://github.com/qoega)). +* Fixed MemorySanitizer errors in cyrus-sasl and musl. [#19821](https://github.com/ClickHouse/ClickHouse/pull/19821) ([Ilya Yatsishin](https://github.com/qoega)). +* Add test for throwing an exception on inserting incorrect data in CollapsingMergeTree. [#19851](https://github.com/ClickHouse/ClickHouse/pull/19851) ([Kruglov Pavel](https://github.com/Avogar)). +* Adding retries for docker-compose start, stop and restart in TestFlows tests. [#19852](https://github.com/ClickHouse/ClickHouse/pull/19852) ([vzakaznikov](https://github.com/vzakaznikov)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Remove useless codes'. [#19293](https://github.com/ClickHouse/ClickHouse/pull/19293) ([sundyli](https://github.com/sundy-li)). +* NO CL ENTRY: 'Merging [#19387](https://github.com/ClickHouse/ClickHouse/issues/19387)'. [#19683](https://github.com/ClickHouse/ClickHouse/pull/19683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.2.3.15-stable.md b/docs/changelogs/v21.2.3.15-stable.md new file mode 100644 index 00000000000..26653e780bb --- /dev/null +++ b/docs/changelogs/v21.2.3.15-stable.md @@ -0,0 +1,19 @@ +### ClickHouse release v21.2.3.15-stable FIXME as compared to v21.2.2.8-stable + +#### Bug Fix +* Backported in [#20241](https://github.com/ClickHouse/ClickHouse/issues/20241): Fix a bug that moving pieces to destination table may failed in case of launching multiple clickhouse-copiers. [#19743](https://github.com/ClickHouse/ClickHouse/pull/19743) ([madianjun](https://github.com/mdianjun)). +* Backported in [#20296](https://github.com/ClickHouse/ClickHouse/issues/20296): * Bugfix in StorageJoin. [#20079](https://github.com/ClickHouse/ClickHouse/pull/20079) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#20389](https://github.com/ClickHouse/ClickHouse/issues/20389): The `MongoDB` table engine now establishes connection only when it's going to read data. `ATTACH TABLE` won't try to connect anymore. [#20110](https://github.com/ClickHouse/ClickHouse/pull/20110) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#20270](https://github.com/ClickHouse/ClickHouse/issues/20270): Fix CTE when using in INSERT SELECT. This fixes [#20187](https://github.com/ClickHouse/ClickHouse/issues/20187), fixes [#20195](https://github.com/ClickHouse/ClickHouse/issues/20195). [#20211](https://github.com/ClickHouse/ClickHouse/pull/20211) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#20329](https://github.com/ClickHouse/ClickHouse/issues/20329): Fix rare server crash on config reload during the shutdown. Fixes [#19689](https://github.com/ClickHouse/ClickHouse/issues/19689). [#20224](https://github.com/ClickHouse/ClickHouse/pull/20224) ([alesapin](https://github.com/alesapin)). +* Backported in [#20324](https://github.com/ClickHouse/ClickHouse/issues/20324): Fix exception during vertical merge for `MergeTree` table engines family which don't allow to perform vertical merges. Fixes [#20259](https://github.com/ClickHouse/ClickHouse/issues/20259). [#20279](https://github.com/ClickHouse/ClickHouse/pull/20279) ([alesapin](https://github.com/alesapin)). +* Backported in [#20333](https://github.com/ClickHouse/ClickHouse/issues/20333): Restrict to `DROP` or `RENAME` version column of `*CollapsingMergeTree` and `ReplacingMergeTree` table engines. [#20300](https://github.com/ClickHouse/ClickHouse/pull/20300) ([alesapin](https://github.com/alesapin)). +* Backported in [#20365](https://github.com/ClickHouse/ClickHouse/issues/20365): Fix too often retries of failed background tasks for `ReplicatedMergeTree` table engines family. This could lead to too verbose logging and increased CPU load. Fixes [#20203](https://github.com/ClickHouse/ClickHouse/issues/20203). [#20335](https://github.com/ClickHouse/ClickHouse/pull/20335) ([alesapin](https://github.com/alesapin)). +* Backported in [#20379](https://github.com/ClickHouse/ClickHouse/issues/20379): Fix incorrect result of binary operations between two constant decimals of different scale. Fixes [#20283](https://github.com/ClickHouse/ClickHouse/issues/20283). [#20339](https://github.com/ClickHouse/ClickHouse/pull/20339) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#20376](https://github.com/ClickHouse/ClickHouse/issues/20376): Fix null dereference with `join_use_nulls=1`. [#20344](https://github.com/ClickHouse/ClickHouse/pull/20344) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#20361](https://github.com/ClickHouse/ClickHouse/issues/20361): Avoid invalid dereference in RANGE_HASHED() dictionary. [#20345](https://github.com/ClickHouse/ClickHouse/pull/20345) ([Azat Khuzhin](https://github.com/azat)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Backport [#20224](https://github.com/ClickHouse/ClickHouse/issues/20224) to 21.2: Fix access control manager destruction order"'. [#20397](https://github.com/ClickHouse/ClickHouse/pull/20397) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.2.4.6-stable.md b/docs/changelogs/v21.2.4.6-stable.md new file mode 100644 index 00000000000..1605186701d --- /dev/null +++ b/docs/changelogs/v21.2.4.6-stable.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.2.4.6-stable FIXME as compared to v21.2.3.15-stable + +#### Bug Fix +* Backported in [#20510](https://github.com/ClickHouse/ClickHouse/issues/20510): Fixed the behavior when in case of broken JSON we tried to read the whole file into memory which leads to exception from the allocator. Fixes [#19719](https://github.com/ClickHouse/ClickHouse/issues/19719). [#20286](https://github.com/ClickHouse/ClickHouse/pull/20286) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#20575](https://github.com/ClickHouse/ClickHouse/issues/20575): Check if table function `view` is used in expression list and throw an error. This fixes [#20342](https://github.com/ClickHouse/ClickHouse/issues/20342). [#20350](https://github.com/ClickHouse/ClickHouse/pull/20350) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#20486](https://github.com/ClickHouse/ClickHouse/issues/20486): Fix `LOGICAL_ERROR` for `join_use_nulls=1` when JOIN contains const from SELECT. [#20461](https://github.com/ClickHouse/ClickHouse/pull/20461) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#20535](https://github.com/ClickHouse/ClickHouse/issues/20535): Fix infinite loop when propagating WITH aliases to subqueries. This fixes [#20388](https://github.com/ClickHouse/ClickHouse/issues/20388). [#20476](https://github.com/ClickHouse/ClickHouse/pull/20476) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#20615](https://github.com/ClickHouse/ClickHouse/issues/20615): Add proper checks while parsing directory names for async INSERT (fixes SIGSEGV). [#20498](https://github.com/ClickHouse/ClickHouse/pull/20498) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#20887](https://github.com/ClickHouse/ClickHouse/issues/20887): Fix subquery with union distinct and limit clause. close [#20597](https://github.com/ClickHouse/ClickHouse/issues/20597). [#20610](https://github.com/ClickHouse/ClickHouse/pull/20610) ([flynn](https://github.com/ucasfl)). +* Backported in [#20993](https://github.com/ClickHouse/ClickHouse/issues/20993): Fix usage of `-Distinct` combinator with `-State` combinator in aggregate functions. [#20866](https://github.com/ClickHouse/ClickHouse/pull/20866) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#20987](https://github.com/ClickHouse/ClickHouse/issues/20987): `USE database;` query did not work when using MySQL 5.7 client to connect to ClickHouse server, it's fixed. Fixes [#18926](https://github.com/ClickHouse/ClickHouse/issues/18926). [#20878](https://github.com/ClickHouse/ClickHouse/pull/20878) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.2.5.5-stable.md b/docs/changelogs/v21.2.5.5-stable.md new file mode 100644 index 00000000000..b5275e89519 --- /dev/null +++ b/docs/changelogs/v21.2.5.5-stable.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.2.5.5-stable FIXME as compared to v21.2.4.6-stable + +#### Bug Fix +* Backported in [#20574](https://github.com/ClickHouse/ClickHouse/issues/20574): Fix crash which could happen if unknown packet was received from remove query (was introduced in [#17868](https://github.com/ClickHouse/ClickHouse/issues/17868)). [#20547](https://github.com/ClickHouse/ClickHouse/pull/20547) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21006](https://github.com/ClickHouse/ClickHouse/issues/21006): Fix 'Empty task was returned from async task queue' on query cancellation. [#20881](https://github.com/ClickHouse/ClickHouse/pull/20881) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21132](https://github.com/ClickHouse/ClickHouse/issues/21132): Fixed behaviour, when `ALTER MODIFY COLUMN` created mutation, that will knowingly fail. [#21007](https://github.com/ClickHouse/ClickHouse/pull/21007) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#21251](https://github.com/ClickHouse/ClickHouse/issues/21251): - Block parallel insertions into storage join. [#21009](https://github.com/ClickHouse/ClickHouse/pull/21009) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#21069](https://github.com/ClickHouse/ClickHouse/issues/21069): Out of bound memory access was possible when formatting specifically crafted out of range value of type `DateTime64`. This closes [#20494](https://github.com/ClickHouse/ClickHouse/issues/20494). This closes [#20543](https://github.com/ClickHouse/ClickHouse/issues/20543). [#21023](https://github.com/ClickHouse/ClickHouse/pull/21023) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#21160](https://github.com/ClickHouse/ClickHouse/issues/21160): Fix `input_format_null_as_default` take effective when types are nullable. This fixes [#21116](https://github.com/ClickHouse/ClickHouse/issues/21116) . [#21121](https://github.com/ClickHouse/ClickHouse/pull/21121) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#21228](https://github.com/ClickHouse/ClickHouse/issues/21228): Fixes [#21112](https://github.com/ClickHouse/ClickHouse/issues/21112). Fixed bug that could cause duplicates with insert query (if one of the callbacks came a little too late). [#21138](https://github.com/ClickHouse/ClickHouse/pull/21138) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#21276](https://github.com/ClickHouse/ClickHouse/issues/21276): Fix bug with `join_use_nulls` and joining `TOTALS` from subqueries. This closes [#19362](https://github.com/ClickHouse/ClickHouse/issues/19362) and [#21137](https://github.com/ClickHouse/ClickHouse/issues/21137). [#21248](https://github.com/ClickHouse/ClickHouse/pull/21248) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.2.6.1-stable.md b/docs/changelogs/v21.2.6.1-stable.md new file mode 100644 index 00000000000..1f28c14c485 --- /dev/null +++ b/docs/changelogs/v21.2.6.1-stable.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.2.6.1-stable FIXME as compared to v21.2.5.5-stable + +#### Bug Fix +* Backported in [#21352](https://github.com/ClickHouse/ClickHouse/issues/21352): Fix the number of threads for scalar subqueries and subqueries for index (after [#19007](https://github.com/ClickHouse/ClickHouse/issues/19007) single thread was always used). Fixes [#20457](https://github.com/ClickHouse/ClickHouse/issues/20457), [#20512](https://github.com/ClickHouse/ClickHouse/issues/20512). [#20550](https://github.com/ClickHouse/ClickHouse/pull/20550) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#21262](https://github.com/ClickHouse/ClickHouse/issues/21262): fix default_replica_path and default_replica_name values are useless on Replicated(*)MergeTree engine when the engine needs specify other parameters. [#21060](https://github.com/ClickHouse/ClickHouse/pull/21060) ([mxzlxy](https://github.com/mxzlxy)). +* Backported in [#21156](https://github.com/ClickHouse/ClickHouse/issues/21156): fix bug related to cast tuple to map. Closes [#21029](https://github.com/ClickHouse/ClickHouse/issues/21029). [#21120](https://github.com/ClickHouse/ClickHouse/pull/21120) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#21234](https://github.com/ClickHouse/ClickHouse/issues/21234): Now mutations allowed only for table engines that support them (MergeTree family, Memory, MaterializedView). Other engines will report a more clear error. Fixes [#21168](https://github.com/ClickHouse/ClickHouse/issues/21168). [#21183](https://github.com/ClickHouse/ClickHouse/pull/21183) ([alesapin](https://github.com/alesapin)). +* Backported in [#21427](https://github.com/ClickHouse/ClickHouse/issues/21427): Fix crash in `EXPLAIN` for query with `UNION`. Fixes [#20876](https://github.com/ClickHouse/ClickHouse/issues/20876), [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170). [#21246](https://github.com/ClickHouse/ClickHouse/pull/21246) ([flynn](https://github.com/ucasfl)). +* Backported in [#21301](https://github.com/ClickHouse/ClickHouse/issues/21301): Fix redundant reconnects to ZooKeeper and the possibility of two active sessions for a single clickhouse server. Both problems introduced in [#14678](https://github.com/ClickHouse/ClickHouse/issues/14678). [#21264](https://github.com/ClickHouse/ClickHouse/pull/21264) ([alesapin](https://github.com/alesapin)). +* Backported in [#21553](https://github.com/ClickHouse/ClickHouse/issues/21553): Now `ALTER MODIFY COLUMN` queries will correctly affect changes in partition key, skip indices, TTLs, and so on. Fixes [#13675](https://github.com/ClickHouse/ClickHouse/issues/13675). [#21334](https://github.com/ClickHouse/ClickHouse/pull/21334) ([alesapin](https://github.com/alesapin)). +* Backported in [#21380](https://github.com/ClickHouse/ClickHouse/issues/21380): Fix error `Bad cast from type ... to DB::ColumnLowCardinality` while inserting into table with `LowCardinality` column from `Values` format. Fixes [#21140](https://github.com/ClickHouse/ClickHouse/issues/21140). [#21357](https://github.com/ClickHouse/ClickHouse/pull/21357) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.2.7.11-stable.md b/docs/changelogs/v21.2.7.11-stable.md new file mode 100644 index 00000000000..1f0f94ee0bf --- /dev/null +++ b/docs/changelogs/v21.2.7.11-stable.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.2.7.11-stable FIXME as compared to v21.2.6.1-stable + +#### Bug Fix +* Backported in [#21206](https://github.com/ClickHouse/ClickHouse/issues/21206): Fix the metadata leak when the Replicated*MergeTree with custom (non default) ZooKeeper cluster is dropped. [#21119](https://github.com/ClickHouse/ClickHouse/pull/21119) ([fastio](https://github.com/fastio)). +* Backported in [#21927](https://github.com/ClickHouse/ClickHouse/issues/21927): Fix Avro format parsing for Kafka. Fixes [#21437](https://github.com/ClickHouse/ClickHouse/issues/21437). [#21438](https://github.com/ClickHouse/ClickHouse/pull/21438) ([Ilya Golshtein](https://github.com/ilejn)). +* Backported in [#21855](https://github.com/ClickHouse/ClickHouse/issues/21855): Fix possible error ` Cannot find column` when `optimize_skip_unused_shards` is enabled and zero shards are used. [#21579](https://github.com/ClickHouse/ClickHouse/pull/21579) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21688](https://github.com/ClickHouse/ClickHouse/issues/21688): Fix fsync_part_directory for horizontal merge. [#21642](https://github.com/ClickHouse/ClickHouse/pull/21642) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21795](https://github.com/ClickHouse/ClickHouse/issues/21795): Fix distributed requests cancellation (for example simple select from multiple shards with limit, i.e. `select * from remote('127.{2,3}', system.numbers) limit 100`) with `async_socket_for_remote=1`. [#21643](https://github.com/ClickHouse/ClickHouse/pull/21643) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21879](https://github.com/ClickHouse/ClickHouse/issues/21879): Fix possible crashes in aggregate functions with combinator Distinct, while using two-level aggregation. This is a follow-up fix of https://github.com/ClickHouse/ClickHouse/pull/18365 . Can only reproduced in production env. No test case available yet. cc @CurtizJ. [#21818](https://github.com/ClickHouse/ClickHouse/pull/21818) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#21979](https://github.com/ClickHouse/ClickHouse/issues/21979): Reverted [#15454](https://github.com/ClickHouse/ClickHouse/issues/15454) that may cause significant increase in memory usage while loading external dictionaries of hashed type. This closes [#21935](https://github.com/ClickHouse/ClickHouse/issues/21935). [#21948](https://github.com/ClickHouse/ClickHouse/pull/21948) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#22140](https://github.com/ClickHouse/ClickHouse/issues/22140): The function `decrypt` was lacking a check for the minimal size of data encrypted in AEAD mode. This closes [#21897](https://github.com/ClickHouse/ClickHouse/issues/21897). [#22064](https://github.com/ClickHouse/ClickHouse/pull/22064) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.2.8.31-stable.md b/docs/changelogs/v21.2.8.31-stable.md new file mode 100644 index 00000000000..884dcb5a649 --- /dev/null +++ b/docs/changelogs/v21.2.8.31-stable.md @@ -0,0 +1,30 @@ +### ClickHouse release v21.2.8.31-stable FIXME as compared to v21.2.7.11-stable + +#### Bug Fix +* Backported in [#22191](https://github.com/ClickHouse/ClickHouse/issues/22191): Fixed race on SSL object inside SecureSocket in Poco. [#21456](https://github.com/ClickHouse/ClickHouse/pull/21456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#22336](https://github.com/ClickHouse/ClickHouse/issues/22336): Fix table function `clusterAllReplicas` returns wrong `_shard_num`. close [#21481](https://github.com/ClickHouse/ClickHouse/issues/21481). [#21498](https://github.com/ClickHouse/ClickHouse/pull/21498) ([flynn](https://github.com/ucasfl)). +* Backported in [#22092](https://github.com/ClickHouse/ClickHouse/issues/22092): In case if query has constant `WHERE` condition, and setting `optimize_skip_unused_shards` enabled, all shards may be skipped and query could return incorrect empty result. [#21550](https://github.com/ClickHouse/ClickHouse/pull/21550) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#22318](https://github.com/ClickHouse/ClickHouse/issues/22318): Remove unknown columns from joined table in where for queries to external database engines (MySQL, PostgreSQL). close [#14614](https://github.com/ClickHouse/ClickHouse/issues/14614), close [#19288](https://github.com/ClickHouse/ClickHouse/issues/19288) (dup), close [#19645](https://github.com/ClickHouse/ClickHouse/issues/19645) (dup). [#21640](https://github.com/ClickHouse/ClickHouse/pull/21640) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#22285](https://github.com/ClickHouse/ClickHouse/issues/22285): Start accepting connections after DDLWorker and dictionaries initialization. [#21676](https://github.com/ClickHouse/ClickHouse/pull/21676) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21753](https://github.com/ClickHouse/ClickHouse/issues/21753): Fix concurrent `OPTIMIZE` and `DROP` for `ReplicatedMergeTree`. [#21716](https://github.com/ClickHouse/ClickHouse/pull/21716) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21809](https://github.com/ClickHouse/ClickHouse/issues/21809): Fix bug for ReplicatedMerge table engines when `ALTER MODIFY COLUMN` query doesn't change the type of decimal column if its size (32 bit or 64 bit) doesn't change. [#21728](https://github.com/ClickHouse/ClickHouse/pull/21728) ([alesapin](https://github.com/alesapin)). +* Backported in [#22188](https://github.com/ClickHouse/ClickHouse/issues/22188): Reverted S3 connection pools. [#21737](https://github.com/ClickHouse/ClickHouse/pull/21737) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#22240](https://github.com/ClickHouse/ClickHouse/issues/22240): Fix scalar subquery index analysis. This fixes [#21717](https://github.com/ClickHouse/ClickHouse/issues/21717) , which was introduced in https://github.com/ClickHouse/ClickHouse/pull/18896 . [#21766](https://github.com/ClickHouse/ClickHouse/pull/21766) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#22050](https://github.com/ClickHouse/ClickHouse/issues/22050): Fix deadlock in first catboost model execution. Closes [#13832](https://github.com/ClickHouse/ClickHouse/issues/13832). [#21844](https://github.com/ClickHouse/ClickHouse/pull/21844) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#22464](https://github.com/ClickHouse/ClickHouse/issues/22464): In rare case, merge for `CollapsingMergeTree` may create granule with `index_granularity + 1` rows. Because of this, internal check, added in [#18928](https://github.com/ClickHouse/ClickHouse/issues/18928) (affects 21.2 and 21.3), may fail with error `Incomplete granules are not allowed while blocks are granules size`. This error did not allow parts to merge. [#21976](https://github.com/ClickHouse/ClickHouse/pull/21976) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#22204](https://github.com/ClickHouse/ClickHouse/issues/22204): Docker entrypoint: avoid chown of `.` in case when `LOG_PATH` is empty. Closes [#22100](https://github.com/ClickHouse/ClickHouse/issues/22100). [#22102](https://github.com/ClickHouse/ClickHouse/pull/22102) ([filimonov](https://github.com/filimonov)). +* Backported in [#22278](https://github.com/ClickHouse/ClickHouse/issues/22278): Fix waiting for `OPTIMIZE` and `ALTER` queries for `ReplicatedMergeTree` table engines. Now the query will not hang when the table was detached or restarted. [#22118](https://github.com/ClickHouse/ClickHouse/pull/22118) ([alesapin](https://github.com/alesapin)). +* Backported in [#22266](https://github.com/ClickHouse/ClickHouse/issues/22266): Fix the background thread pool name. [#22122](https://github.com/ClickHouse/ClickHouse/pull/22122) ([fastio](https://github.com/fastio)). +* Backported in [#22315](https://github.com/ClickHouse/ClickHouse/issues/22315): Fix docker entrypoint in case `http_port` is not in the config. [#22132](https://github.com/ClickHouse/ClickHouse/pull/22132) ([Ewout](https://github.com/devwout)). +* Backported in [#22502](https://github.com/ClickHouse/ClickHouse/issues/22502): Fix query cancellation with `use_hedged_requests=0` and `async_socket_for_remote=1`. [#22183](https://github.com/ClickHouse/ClickHouse/pull/22183) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22369](https://github.com/ClickHouse/ClickHouse/issues/22369): Now clickhouse will not throw `LOGICAL_ERROR` exception when we try to mutate the already covered part. Fixes [#22013](https://github.com/ClickHouse/ClickHouse/issues/22013). [#22291](https://github.com/ClickHouse/ClickHouse/pull/22291) ([alesapin](https://github.com/alesapin)). +* Backported in [#22530](https://github.com/ClickHouse/ClickHouse/issues/22530): Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22461](https://github.com/ClickHouse/ClickHouse/issues/22461): Add (missing) memory accounting in parallel parsing routines. In previous versions OOM was possible when the resultset contains very large blocks of data. This closes [#22008](https://github.com/ClickHouse/ClickHouse/issues/22008). [#22425](https://github.com/ClickHouse/ClickHouse/pull/22425) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22557](https://github.com/ClickHouse/ClickHouse/issues/22557): Fix bug in partial merge join with `LowCardinality`. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#22606](https://github.com/ClickHouse/ClickHouse/issues/22606): Fix deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22575](https://github.com/ClickHouse/ClickHouse/issues/22575): Fix UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22648](https://github.com/ClickHouse/ClickHouse/issues/22648): Avoid UB in *Log engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22682](https://github.com/ClickHouse/ClickHouse/issues/22682): Fix LOGICAL_ERROR for Log with nested types w/o columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22699](https://github.com/ClickHouse/ClickHouse/issues/22699): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). +* Backported in [#22740](https://github.com/ClickHouse/ClickHouse/issues/22740): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.2.9.41-stable.md b/docs/changelogs/v21.2.9.41-stable.md new file mode 100644 index 00000000000..ab4303aaa2a --- /dev/null +++ b/docs/changelogs/v21.2.9.41-stable.md @@ -0,0 +1,17 @@ +### ClickHouse release v21.2.9.41-stable FIXME as compared to v21.2.8.31-stable + +#### Improvement +* Backported in [#22818](https://github.com/ClickHouse/ClickHouse/issues/22818): Make FQDN and other DNS related functions work correctly in alpine images. [#20336](https://github.com/ClickHouse/ClickHouse/pull/20336) ([filimonov](https://github.com/filimonov)). +* Backported in [#22812](https://github.com/ClickHouse/ClickHouse/issues/22812): If PODArray was instantiated with element size that is neither a fraction or a multiple of 16, buffer overflow was possible. No bugs in current releases exist. [#21533](https://github.com/ClickHouse/ClickHouse/pull/21533) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#22965](https://github.com/ClickHouse/ClickHouse/issues/22965): Fix very rare bug when quorum insert with `quorum_parallel=1` is not really "quorum" because of deduplication. [#18215](https://github.com/ClickHouse/ClickHouse/pull/18215) ([filimonov](https://github.com/filimonov)). +* Backported in [#22719](https://github.com/ClickHouse/ClickHouse/issues/22719): Check if table function view is used as a column. This complements https://github.com/ClickHouse/ClickHouse/pull/20350. [#21465](https://github.com/ClickHouse/ClickHouse/pull/21465) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#22756](https://github.com/ClickHouse/ClickHouse/issues/22756): Fix usage of function `map` in distributed queries. [#22588](https://github.com/ClickHouse/ClickHouse/pull/22588) ([foolchi](https://github.com/foolchi)). +* Backported in [#22889](https://github.com/ClickHouse/ClickHouse/issues/22889): Fix approx total rows accounting for reverse reading from MergeTree. [#22726](https://github.com/ClickHouse/ClickHouse/pull/22726) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22915](https://github.com/ClickHouse/ClickHouse/issues/22915): LIVE VIEW (experimental feature). Fix possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, see https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e. [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#22918](https://github.com/ClickHouse/ClickHouse/issues/22918): Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### Build/Testing/Packaging Improvement +* Backported in [#22814](https://github.com/ClickHouse/ClickHouse/issues/22814): Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.3.1.6185-prestable.md b/docs/changelogs/v21.3.1.6185-prestable.md new file mode 100644 index 00000000000..dabfe3cfeb3 --- /dev/null +++ b/docs/changelogs/v21.3.1.6185-prestable.md @@ -0,0 +1,159 @@ +### ClickHouse release v21.3.1.6185-prestable FIXME as compared to v21.2.1.5869-prestable + +#### Backward Incompatible Change +* Now all case-insensitive function names will be lower-cased during query analysis. This is needed for projection query routing. [#20174](https://github.com/ClickHouse/ClickHouse/pull/20174) ([Amos Bird](https://github.com/amosbird)). +* Now it's not allowed to create MergeTree tables in old syntax with table TTL because it's just ignored. Attach of old tables is still possible. [#20282](https://github.com/ClickHouse/ClickHouse/pull/20282) ([alesapin](https://github.com/alesapin)). + +#### New Feature +* Add experimental `Replicated` database engine. It replicates DDL queries across multiple hosts. [#16193](https://github.com/ClickHouse/ClickHouse/pull/16193) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Distributed query deduplication is a followup to [#16033](https://github.com/ClickHouse/ClickHouse/issues/16033) and partially resolves the proposal [#13574](https://github.com/ClickHouse/ClickHouse/issues/13574). [#17348](https://github.com/ClickHouse/ClickHouse/pull/17348) ([xjewer](https://github.com/xjewer)). +* Add the ability to backup/restore metadata files for DiskS3. [#18377](https://github.com/ClickHouse/ClickHouse/pull/18377) ([Pavel Kovalenko](https://github.com/Jokser)). +* - Included in the pull request. [#18508](https://github.com/ClickHouse/ClickHouse/pull/18508) ([PHO](https://github.com/depressed-pho)). +* Added file() function to read file as a String. This close [#issue:18851](https://github.com/ClickHouse/ClickHouse/issues/18851). [#19204](https://github.com/ClickHouse/ClickHouse/pull/19204) ([keenwolf](https://github.com/keen-wolf)). +* Tables with `MergeTree*` engine now have two new table-level settings for query concurrency control. Setting `max_concurrent_queries` limits the number of concurrently executed queries which are related to this table. Setting `min_marks_to_honor_max_concurrent_queries` tells to apply previous setting only if query reads at least this number of marks. [#19544](https://github.com/ClickHouse/ClickHouse/pull/19544) ([Amos Bird](https://github.com/amosbird)). +* - Mentioned in [#18454](https://github.com/ClickHouse/ClickHouse/issues/18454) - add function `htmlOrxmlCoarseParse`; - support `` parse; - support `` parse; - support `` parse; - support white space collapse; - support any `` format parse; - HyperScan to support SIMD; - Everything is done in a single pass. [#19600](https://github.com/ClickHouse/ClickHouse/pull/19600) ([zlx19950903](https://github.com/zlx19950903)). +* Add quota type QUERY_SELECTS and QUERY_INSERTS. [#19603](https://github.com/ClickHouse/ClickHouse/pull/19603) ([JackyWoo](https://github.com/JackyWoo)). +* ExecutableDictionarySource added implicit_key option. Fixes [#14527](https://github.com/ClickHouse/ClickHouse/issues/14527). [#19677](https://github.com/ClickHouse/ClickHouse/pull/19677) ([Maksim Kita](https://github.com/kitaisreal)). +* Added Server Side Encryption Customer Keys (the `x-amz-server-side-encryption-customer-(key/md5)` header) support in S3 client. See [the link](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). Closes [#19428](https://github.com/ClickHouse/ClickHouse/issues/19428). [#19748](https://github.com/ClickHouse/ClickHouse/pull/19748) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Function `reinterpretAs` updated to support big integers. Fixes [#19691](https://github.com/ClickHouse/ClickHouse/issues/19691). [#19858](https://github.com/ClickHouse/ClickHouse/pull/19858) ([Maksim Kita](https://github.com/kitaisreal)). +* Add setting `insert_shard_id` to support insert data into specific shard from distributed table. [#19961](https://github.com/ClickHouse/ClickHouse/pull/19961) ([flynn](https://github.com/ucasfl)). +* Added timezoneOffset(datetime) function which will give the offset from UTC in seconds. This close [#issue:19850](https://github.com/ClickHouse/ClickHouse/issues/19850). [#19962](https://github.com/ClickHouse/ClickHouse/pull/19962) ([keenwolf](https://github.com/keen-wolf)). +* New `event_time_microseconds column` in `system.part_log` table. [#20027](https://github.com/ClickHouse/ClickHouse/pull/20027) ([Bharat Nallan](https://github.com/bharatnc)). +* ... Add aggregate function `deltaSum` for summing the differences between consecutive rows. [#20057](https://github.com/ClickHouse/ClickHouse/pull/20057) ([Russ Frank](https://github.com/rf)). +* Add two settings to delay or throw error during insertion when there are too many inactive parts. This is useful when server fails to clean up parts quickly enough. [#20178](https://github.com/ClickHouse/ClickHouse/pull/20178) ([Amos Bird](https://github.com/amosbird)). +* Add file engine settings: `engine_file_empty_if_not_exists` and `engine_file_truncate_on_insert`. [#20620](https://github.com/ClickHouse/ClickHouse/pull/20620) ([M0r64n](https://github.com/M0r64n)). + +#### Performance Improvement +* Add parallel select final for one part with level>0 when `do_not_merge_across_partitions_select_final` setting is 1. [#19375](https://github.com/ClickHouse/ClickHouse/pull/19375) ([Kruglov Pavel](https://github.com/Avogar)). +* Improved performance of bitmap columns during joins. [#19407](https://github.com/ClickHouse/ClickHouse/pull/19407) ([templarzq](https://github.com/templarzq)). +* Partially reimplement HTTP server to make it making less copies of incoming and outgoing data. It gives up to 1.5 performance improvement on inserting long records over HTTP. [#19516](https://github.com/ClickHouse/ClickHouse/pull/19516) ([Ivan](https://github.com/abyss7)). +* Improve performance of aggregate functions by more strict aliasing. [#19946](https://github.com/ClickHouse/ClickHouse/pull/19946) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix the case when DataType parser may have exponential complexity (found by fuzzer). This closes [#20096](https://github.com/ClickHouse/ClickHouse/issues/20096). [#20132](https://github.com/ClickHouse/ClickHouse/pull/20132) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `compress` setting for `Memory` tables. If it's enabled the table will use less RAM. On some machines and datasets it can also work faster on SELECT, but it is not always the case. This closes [#20093](https://github.com/ClickHouse/ClickHouse/issues/20093). Note: there are reasons why Memory tables can work slower than MergeTree: (1) lack of compression (2) static size of blocks (3) lack of indices and prewhere... [#20168](https://github.com/ClickHouse/ClickHouse/pull/20168) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not squash blocks too much on INSERT SELECT if inserting into Memory table. In previous versions inefficient data representation was created in Memory table after INSERT SELECT. This closes [#13052](https://github.com/ClickHouse/ClickHouse/issues/13052). [#20169](https://github.com/ClickHouse/ClickHouse/pull/20169) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improved performance of aggregation by several fixed size fields (unconfirmed). [#20454](https://github.com/ClickHouse/ClickHouse/pull/20454) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Speed up reading from `Memory` tables in extreme cases (when reading speed is in order of 50 GB/sec) by simplification of pipeline and (consequently) less lock contention in pipeline scheduling. [#20468](https://github.com/ClickHouse/ClickHouse/pull/20468) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve performance of GROUP BY multiple fixed size keys. [#20472](https://github.com/ClickHouse/ClickHouse/pull/20472) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The setting `distributed_aggregation_memory_efficient` is enabled by default. It will lower memory usage and improve performance of distributed queries. [#20599](https://github.com/ClickHouse/ClickHouse/pull/20599) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Slightly better code in aggregation. [#20978](https://github.com/ClickHouse/ClickHouse/pull/20978) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add back intDiv/module vectorConstant specializations for better performance. This fixes [#21293](https://github.com/ClickHouse/ClickHouse/issues/21293) . The regression was introduced in https://github.com/ClickHouse/ClickHouse/pull/18145 . [#21307](https://github.com/ClickHouse/ClickHouse/pull/21307) ([Amos Bird](https://github.com/amosbird)). + +#### Improvement +* Fix creation of `TTL` in cases, when its expression is a function and it is the same as `ORDER BY` key. Now it's allowed to set custom aggregation to primary key columns in `TTL` with `GROUP BY`. Backward incompatible: For primary key columns, which are not in `GROUP BY` and aren't set explicitly now is applied function `any` instead of `max`, when TTL is expired. Also if you use TTL with `WHERE` or `GROUP BY` you can see exceptions at merges, while making rolling update. [#15450](https://github.com/ClickHouse/ClickHouse/pull/15450) ([Anton Popov](https://github.com/CurtizJ)). +* Hedged Requests for remote queries. When setting `use_hedged_requests` enabled (by default), allow to establish many connections with different replicas for query. New connection is enabled in case existent connection(s) with replica(s) were not established within `hedged_connection_timeout` or no data was received within `receive_data_timeout`. Query uses the first connection which send non empty progress packet (or data packet, if `allow_changing_replica_until_first_data_packet`); other connections are cancelled. Queries with `max_parallel_replicas > 1` are supported. [#19291](https://github.com/ClickHouse/ClickHouse/pull/19291) ([Kruglov Pavel](https://github.com/Avogar)). +* Print inline frames for fatal stacktraces. [#19317](https://github.com/ClickHouse/ClickHouse/pull/19317) ([Ivan](https://github.com/abyss7)). +* Do not silently ignore write errors. [#19451](https://github.com/ClickHouse/ClickHouse/pull/19451) ([Azat Khuzhin](https://github.com/azat)). +* Added support for `PREWHERE` when tables have row-level security expressions specified. [#19576](https://github.com/ClickHouse/ClickHouse/pull/19576) ([Denis Glazachev](https://github.com/traceon)). +* Add IStoragePolicy interface. [#19608](https://github.com/ClickHouse/ClickHouse/pull/19608) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). +* Add ability to throttle INSERT into Distributed based on amount of pending bytes for async send (`bytes_to_delay_insert`/`max_delay_to_insert` and `bytes_to_throw_insert` settings for `Distributed` engine has been added). [#19673](https://github.com/ClickHouse/ClickHouse/pull/19673) ([Azat Khuzhin](https://github.com/azat)). +* move Conditions that are not related to JOIN to where clause. [#18720](https://github.com/ClickHouse/ClickHouse/issues/18720). [#19685](https://github.com/ClickHouse/ClickHouse/pull/19685) ([hexiaoting](https://github.com/hexiaoting)). +* Add separate config directive for Buffer profile. [#19721](https://github.com/ClickHouse/ClickHouse/pull/19721) ([Azat Khuzhin](https://github.com/azat)). +* Show MaterializeMySQL tables in `system.parts`. [#19770](https://github.com/ClickHouse/ClickHouse/pull/19770) ([Stig Bakken](https://github.com/stigsb)). +* Initialize MaxDDLEntryID to the last value after restarting. Before this PR, MaxDDLEntryID will remain zero until a new DDLTask is processed. [#19924](https://github.com/ClickHouse/ClickHouse/pull/19924) ([Amos Bird](https://github.com/amosbird)). +* Add conversion of block structure for INSERT into Distributed tables if it does not match. [#19947](https://github.com/ClickHouse/ClickHouse/pull/19947) ([Azat Khuzhin](https://github.com/azat)). +* If user calls `JSONExtract` function with `Float32` type requested, allow inaccurate conversion to the result type. For example the number `0.1` in JSON is double precision and is not representable in Float32, but the user still wants to get it. Previous versions return 0 for non-Nullable type and NULL for Nullable type to indicate that conversion is imprecise. The logic was 100% correct but it was surprising to users and leading to questions. This closes [#13962](https://github.com/ClickHouse/ClickHouse/issues/13962). [#19960](https://github.com/ClickHouse/ClickHouse/pull/19960) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The value of MYSQL_OPT_RECONNECT option can now be controlled by "opt_reconnect" parameter in the config section of mysql replica. [#19998](https://github.com/ClickHouse/ClickHouse/pull/19998) ([Alexander Kazakov](https://github.com/Akazz)). +* Return `DiskType` instead of `String` in IDisk::getType() as in the rest of storage interfaces. [#19999](https://github.com/ClickHouse/ClickHouse/pull/19999) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). +* Fix data race in executable dictionary that was possible only on misuse (when the script returns data ignoring its input). [#20045](https://github.com/ClickHouse/ClickHouse/pull/20045) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Show full details of MaterializeMySQL tables in `system.tables`. [#20051](https://github.com/ClickHouse/ClickHouse/pull/20051) ([Stig Bakken](https://github.com/stigsb)). +* Supports system.zookeeper path IN query. [#20105](https://github.com/ClickHouse/ClickHouse/pull/20105) ([小路](https://github.com/nicelulu)). +* 1. SHOW TABLES is now considered as one query in the quota calculations, not two queries. 2. SYSTEM queries now consume quota. 3. Fix calculation of interval's end in quota consumption. [#20106](https://github.com/ClickHouse/ClickHouse/pull/20106) ([Vitaly Baranov](https://github.com/vitlibar)). +* - Fix toDateTime64(toDate()/toDateTime()) for DateTime64 - Implement DateTime64 clamping to match DateTime behaviour. [#20131](https://github.com/ClickHouse/ClickHouse/pull/20131) ([Azat Khuzhin](https://github.com/azat)). +* The setting `access_management` is now configurable on startup by providing `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT`, defaults to disabled (`0`) which was the prior value. [#20139](https://github.com/ClickHouse/ClickHouse/pull/20139) ([Marquitos](https://github.com/sonirico)). +* Updated `CacheDictionary`, `ComplexCacheDictionary`, `SSDCacheDictionary`, `SSDComplexKeyDictionary` to use LRUHashMap as underlying index. [#20164](https://github.com/ClickHouse/ClickHouse/pull/20164) ([Maksim Kita](https://github.com/kitaisreal)). +* Support all native integer types in bitmap functions. [#20171](https://github.com/ClickHouse/ClickHouse/pull/20171) ([Amos Bird](https://github.com/amosbird)). +* Normalize count(constant), sum(1) to count(). This is needed for projection query routing. [#20175](https://github.com/ClickHouse/ClickHouse/pull/20175) ([Amos Bird](https://github.com/amosbird)). +* Perform algebraic optimizations of arithmetic expressions inside `avg` aggregate function. close [#20092](https://github.com/ClickHouse/ClickHouse/issues/20092). [#20183](https://github.com/ClickHouse/ClickHouse/pull/20183) ([flynn](https://github.com/ucasfl)). +* Lockless `SYSTEM FLUSH DISTRIBUTED`. [#20215](https://github.com/ClickHouse/ClickHouse/pull/20215) ([Azat Khuzhin](https://github.com/azat)). +* Implicit conversion from integer to Dicimal type might succeeded if integer value doe not fit into Decimal type. Now it throws `ARGUMENT_OUT_OF_BOUND`. [#20232](https://github.com/ClickHouse/ClickHouse/pull/20232) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Do not allow early constant folding of explicitly forbidden functions. [#20303](https://github.com/ClickHouse/ClickHouse/pull/20303) ([Azat Khuzhin](https://github.com/azat)). +* Make FQDN and other DNS related functions work correctly in alpine images. [#20336](https://github.com/ClickHouse/ClickHouse/pull/20336) ([filimonov](https://github.com/filimonov)). +* Fixed race between execution of distributed DDL tasks and cleanup of DDL queue. Now DDL task cannot be removed from ZooKeeper if there are active workers. Fixes [#20016](https://github.com/ClickHouse/ClickHouse/issues/20016). [#20448](https://github.com/ClickHouse/ClickHouse/pull/20448) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improved serialization for data types combined of Arrays and Tuples. Improved matching enum data types to protobuf enum type. Fixed serialization of the `Map` data type. Omitted values are now set by default. [#20506](https://github.com/ClickHouse/ClickHouse/pull/20506) ([Vitaly Baranov](https://github.com/vitlibar)). +* https://github.com/ClickHouse/ClickHouse/issues/20576. [#20596](https://github.com/ClickHouse/ClickHouse/pull/20596) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Function 'reinterpretAs(x, Type)' renamed into 'reinterpret(x, Type)'. [#20611](https://github.com/ClickHouse/ClickHouse/pull/20611) ([Maksim Kita](https://github.com/kitaisreal)). +* When loading config for mysql source ClickHouse will now randomize the list of replicas with the same priority to ensure the round-robin logics of picking mysql endpoint. This closes [#20629](https://github.com/ClickHouse/ClickHouse/issues/20629). [#20632](https://github.com/ClickHouse/ClickHouse/pull/20632) ([Alexander Kazakov](https://github.com/Akazz)). +* Do only merging of sorted blocks on initiator with distributed_group_by_no_merge. [#20882](https://github.com/ClickHouse/ClickHouse/pull/20882) ([Azat Khuzhin](https://github.com/azat)). +* - Fill only requested columns when querying system.parts & system.parts_columns. Closes [#19570](https://github.com/ClickHouse/ClickHouse/issues/19570). ... [#21035](https://github.com/ClickHouse/ClickHouse/pull/21035) ([Anmol Arora](https://github.com/anmolarora)). +* Usability improvement: more consistent `DateTime64` parsing: recognize the case when unix timestamp with subsecond resolution is specified as scaled integer (like `1111111111222` instead of `1111111111.222`). This closes [#13194](https://github.com/ClickHouse/ClickHouse/issues/13194). [#21053](https://github.com/ClickHouse/ClickHouse/pull/21053) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* MySQL dictionary source will now retry unexpected connection failures (Lost connection to MySQL server during query) which sometimes happen on SSL/TLS connections. [#21237](https://github.com/ClickHouse/ClickHouse/pull/21237) ([Alexander Kazakov](https://github.com/Akazz)). +* Forbid to drop a column if it's referenced by materialized view. Closes [#21164](https://github.com/ClickHouse/ClickHouse/issues/21164). [#21303](https://github.com/ClickHouse/ClickHouse/pull/21303) ([flynn](https://github.com/ucasfl)). +* Provide better compatibility for mysql clients. 1. mysql jdbc 2. mycli. [#21367](https://github.com/ClickHouse/ClickHouse/pull/21367) ([Amos Bird](https://github.com/amosbird)). +* Case-insensitive compression methods for table functions. Also fixed `LZMA` compression method which was checked in upper case. [#21416](https://github.com/ClickHouse/ClickHouse/pull/21416) ([Vladimir Chebotarev](https://github.com/excitoon)). + +#### Bug Fix +* Background thread which executes `ON CLUSTER` queries might hang waiting for dropped replicated table to do something. It's fixed. [#19684](https://github.com/ClickHouse/ClickHouse/pull/19684) ([yiguolei](https://github.com/yiguolei)). +* Fix a bug that moving pieces to destination table may failed in case of launching multiple clickhouse-copiers. [#19743](https://github.com/ClickHouse/ClickHouse/pull/19743) ([madianjun](https://github.com/mdianjun)). +* Fix clickhouse-client abort exception while executing only `select`. [#19790](https://github.com/ClickHouse/ClickHouse/pull/19790) ([李扬](https://github.com/taiyang-li)). +* Fix starting the server with tables having default expressions containing dictGet(). Allow getting return type of dictGet() without loading dictionary. [#19805](https://github.com/ClickHouse/ClickHouse/pull/19805) ([Vitaly Baranov](https://github.com/vitlibar)). +* Deadlock was possible if system.text_log is enabled. This fixes [#19874](https://github.com/ClickHouse/ClickHouse/issues/19874). [#19875](https://github.com/ClickHouse/ClickHouse/pull/19875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* BloomFilter index crash fix. Fixes [#19757](https://github.com/ClickHouse/ClickHouse/issues/19757). [#19884](https://github.com/ClickHouse/ClickHouse/pull/19884) ([Maksim Kita](https://github.com/kitaisreal)). +* - Fix a segfault in function `fromModifiedJulianDay` when the argument type is `Nullable(T)` for any integral types other than Int32. [#19959](https://github.com/ClickHouse/ClickHouse/pull/19959) ([PHO](https://github.com/depressed-pho)). +* `EmbeddedRocksDB` is an experimental storage. Fix the issue with lack of proper type checking. Simplified code. This closes [#19967](https://github.com/ClickHouse/ClickHouse/issues/19967). [#19972](https://github.com/ClickHouse/ClickHouse/pull/19972) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Prevent "Connection refused" in docker during initialization script execution. [#20012](https://github.com/ClickHouse/ClickHouse/pull/20012) ([filimonov](https://github.com/filimonov)). +* MaterializeMySQL: Fix replication for statements that update several tables. [#20066](https://github.com/ClickHouse/ClickHouse/pull/20066) ([Håvard Kvålen](https://github.com/havardk)). +* Fix the case when calculating modulo of division of negative number by small divisor, the resulting data type was not large enough to accomodate the negative result. This closes [#20052](https://github.com/ClickHouse/ClickHouse/issues/20052). [#20067](https://github.com/ClickHouse/ClickHouse/pull/20067) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* * Bugfix in StorageJoin. [#20079](https://github.com/ClickHouse/ClickHouse/pull/20079) ([Vladimir C](https://github.com/vdimir)). +* The `MongoDB` table engine now establishes connection only when it's going to read data. `ATTACH TABLE` won't try to connect anymore. [#20110](https://github.com/ClickHouse/ClickHouse/pull/20110) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix server crash after query with `if` function with `Tuple` type of then/else branches result. `Tuple` type must contain `Array` or another complex type. Fixes [#18356](https://github.com/ClickHouse/ClickHouse/issues/18356). [#20133](https://github.com/ClickHouse/ClickHouse/pull/20133) ([alesapin](https://github.com/alesapin)). +* fix toMinute function to handle special timezone correctly. [#20149](https://github.com/ClickHouse/ClickHouse/pull/20149) ([keenwolf](https://github.com/keen-wolf)). +* Fixes [#19314](https://github.com/ClickHouse/ClickHouse/issues/19314). [#20156](https://github.com/ClickHouse/ClickHouse/pull/20156) ([Ivan](https://github.com/abyss7)). +* Fix CTE when using in INSERT SELECT. This fixes [#20187](https://github.com/ClickHouse/ClickHouse/issues/20187), fixes [#20195](https://github.com/ClickHouse/ClickHouse/issues/20195). [#20211](https://github.com/ClickHouse/ClickHouse/pull/20211) ([Amos Bird](https://github.com/amosbird)). +* Fix rare server crash on config reload during the shutdown. Fixes [#19689](https://github.com/ClickHouse/ClickHouse/issues/19689). [#20224](https://github.com/ClickHouse/ClickHouse/pull/20224) ([alesapin](https://github.com/alesapin)). +* Fix exception during vertical merge for `MergeTree` table engines family which don't allow to perform vertical merges. Fixes [#20259](https://github.com/ClickHouse/ClickHouse/issues/20259). [#20279](https://github.com/ClickHouse/ClickHouse/pull/20279) ([alesapin](https://github.com/alesapin)). +* Fixed the behavior when in case of broken JSON we tried to read the whole file into memory which leads to exception from the allocator. Fixes [#19719](https://github.com/ClickHouse/ClickHouse/issues/19719). [#20286](https://github.com/ClickHouse/ClickHouse/pull/20286) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Restrict to `DROP` or `RENAME` version column of `*CollapsingMergeTree` and `ReplacingMergeTree` table engines. [#20300](https://github.com/ClickHouse/ClickHouse/pull/20300) ([alesapin](https://github.com/alesapin)). +* Fix too often retries of failed background tasks for `ReplicatedMergeTree` table engines family. This could lead to too verbose logging and increased CPU load. Fixes [#20203](https://github.com/ClickHouse/ClickHouse/issues/20203). [#20335](https://github.com/ClickHouse/ClickHouse/pull/20335) ([alesapin](https://github.com/alesapin)). +* Fix incorrect result of binary operations between two constant decimals of different scale. Fixes [#20283](https://github.com/ClickHouse/ClickHouse/issues/20283). [#20339](https://github.com/ClickHouse/ClickHouse/pull/20339) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix null dereference with `join_use_nulls=1`. [#20344](https://github.com/ClickHouse/ClickHouse/pull/20344) ([Azat Khuzhin](https://github.com/azat)). +* Avoid invalid dereference in RANGE_HASHED() dictionary. [#20345](https://github.com/ClickHouse/ClickHouse/pull/20345) ([Azat Khuzhin](https://github.com/azat)). +* Check if table function `view` is used in expression list and throw an error. This fixes [#20342](https://github.com/ClickHouse/ClickHouse/issues/20342). [#20350](https://github.com/ClickHouse/ClickHouse/pull/20350) ([Amos Bird](https://github.com/amosbird)). +* Fix `LOGICAL_ERROR` for `join_use_nulls=1` when JOIN contains const from SELECT. [#20461](https://github.com/ClickHouse/ClickHouse/pull/20461) ([Azat Khuzhin](https://github.com/azat)). +* Fix abnormal server termination when http client goes away. [#20464](https://github.com/ClickHouse/ClickHouse/pull/20464) ([Azat Khuzhin](https://github.com/azat)). +* Fix infinite loop when propagating WITH aliases to subqueries. This fixes [#20388](https://github.com/ClickHouse/ClickHouse/issues/20388). [#20476](https://github.com/ClickHouse/ClickHouse/pull/20476) ([Amos Bird](https://github.com/amosbird)). +* Fix function `transform` does not work properly for floating point keys. Closes [#20460](https://github.com/ClickHouse/ClickHouse/issues/20460). [#20479](https://github.com/ClickHouse/ClickHouse/pull/20479) ([flynn](https://github.com/ucasfl)). +* Add proper checks while parsing directory names for async INSERT (fixes SIGSEGV). [#20498](https://github.com/ClickHouse/ClickHouse/pull/20498) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash which could happen if unknown packet was received from remove query (was introduced in [#17868](https://github.com/ClickHouse/ClickHouse/issues/17868)). [#20547](https://github.com/ClickHouse/ClickHouse/pull/20547) ([Azat Khuzhin](https://github.com/azat)). +* Fix the number of threads for scalar subqueries and subqueries for index (after [#19007](https://github.com/ClickHouse/ClickHouse/issues/19007) single thread was always used). Fixes [#20457](https://github.com/ClickHouse/ClickHouse/issues/20457), [#20512](https://github.com/ClickHouse/ClickHouse/issues/20512). [#20550](https://github.com/ClickHouse/ClickHouse/pull/20550) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed inconsistent behavior of dictionary in case of queries where we look for absent keys in dictionary. [#20578](https://github.com/ClickHouse/ClickHouse/pull/20578) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix subquery with union distinct and limit clause. close [#20597](https://github.com/ClickHouse/ClickHouse/issues/20597). [#20610](https://github.com/ClickHouse/ClickHouse/pull/20610) ([flynn](https://github.com/ucasfl)). +* Fix usage of `-Distinct` combinator with `-State` combinator in aggregate functions. [#20866](https://github.com/ClickHouse/ClickHouse/pull/20866) ([Anton Popov](https://github.com/CurtizJ)). +* `USE database;` query did not work when using MySQL 5.7 client to connect to ClickHouse server, it's fixed. Fixes [#18926](https://github.com/ClickHouse/ClickHouse/issues/18926). [#20878](https://github.com/ClickHouse/ClickHouse/pull/20878) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 'Empty task was returned from async task queue' on query cancellation. [#20881](https://github.com/ClickHouse/ClickHouse/pull/20881) ([Azat Khuzhin](https://github.com/azat)). +* Closes [#9969](https://github.com/ClickHouse/ClickHouse/issues/9969). Fixed Brotli http compression error, which reproduced for large data sizes, slightly complicated structure and with json output format. Update Brotli to the latest version to include the "fix rare access to uninitialized data in ring-buffer". [#20991](https://github.com/ClickHouse/ClickHouse/pull/20991) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed behaviour, when `ALTER MODIFY COLUMN` created mutation, that will knowingly fail. [#21007](https://github.com/ClickHouse/ClickHouse/pull/21007) ([Anton Popov](https://github.com/CurtizJ)). +* - Block parallel insertions into storage join. [#21009](https://github.com/ClickHouse/ClickHouse/pull/21009) ([Vladimir C](https://github.com/vdimir)). +* Out of bound memory access was possible when formatting specifically crafted out of range value of type `DateTime64`. This closes [#20494](https://github.com/ClickHouse/ClickHouse/issues/20494). This closes [#20543](https://github.com/ClickHouse/ClickHouse/issues/20543). [#21023](https://github.com/ClickHouse/ClickHouse/pull/21023) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix default_replica_path and default_replica_name values are useless on Replicated(*)MergeTree engine when the engine needs specify other parameters. [#21060](https://github.com/ClickHouse/ClickHouse/pull/21060) ([mxzlxy](https://github.com/mxzlxy)). +* Fix type mismatch issue when using LowCardinality keys in joinGet. This fixes [#21114](https://github.com/ClickHouse/ClickHouse/issues/21114). [#21117](https://github.com/ClickHouse/ClickHouse/pull/21117) ([Amos Bird](https://github.com/amosbird)). +* Fix the metadata leak when the Replicated*MergeTree with custom (non default) ZooKeeper cluster is dropped. [#21119](https://github.com/ClickHouse/ClickHouse/pull/21119) ([fastio](https://github.com/fastio)). +* fix bug related to cast tuple to map. Closes [#21029](https://github.com/ClickHouse/ClickHouse/issues/21029). [#21120](https://github.com/ClickHouse/ClickHouse/pull/21120) ([hexiaoting](https://github.com/hexiaoting)). +* Fix `input_format_null_as_default` take effective when types are nullable. This fixes [#21116](https://github.com/ClickHouse/ClickHouse/issues/21116) . [#21121](https://github.com/ClickHouse/ClickHouse/pull/21121) ([Amos Bird](https://github.com/amosbird)). +* Fixes [#21112](https://github.com/ClickHouse/ClickHouse/issues/21112). Fixed bug that could cause duplicates with insert query (if one of the callbacks came a little too late). [#21138](https://github.com/ClickHouse/ClickHouse/pull/21138) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Now mutations allowed only for table engines that support them (MergeTree family, Memory, MaterializedView). Other engines will report a more clear error. Fixes [#21168](https://github.com/ClickHouse/ClickHouse/issues/21168). [#21183](https://github.com/ClickHouse/ClickHouse/pull/21183) ([alesapin](https://github.com/alesapin)). +* Fix crash in `EXPLAIN` for query with `UNION`. Fixes [#20876](https://github.com/ClickHouse/ClickHouse/issues/20876), [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170). [#21246](https://github.com/ClickHouse/ClickHouse/pull/21246) ([flynn](https://github.com/ucasfl)). +* Fix bug with `join_use_nulls` and joining `TOTALS` from subqueries. This closes [#19362](https://github.com/ClickHouse/ClickHouse/issues/19362) and [#21137](https://github.com/ClickHouse/ClickHouse/issues/21137). [#21248](https://github.com/ClickHouse/ClickHouse/pull/21248) ([Vladimir C](https://github.com/vdimir)). +* Fix redundant reconnects to ZooKeeper and the possibility of two active sessions for a single clickhouse server. Both problems introduced in [#14678](https://github.com/ClickHouse/ClickHouse/issues/14678). [#21264](https://github.com/ClickHouse/ClickHouse/pull/21264) ([alesapin](https://github.com/alesapin)). +* Now `ALTER MODIFY COLUMN` queries will correctly affect changes in partition key, skip indices, TTLs, and so on. Fixes [#13675](https://github.com/ClickHouse/ClickHouse/issues/13675). [#21334](https://github.com/ClickHouse/ClickHouse/pull/21334) ([alesapin](https://github.com/alesapin)). +* Fix error `Bad cast from type ... to DB::ColumnLowCardinality` while inserting into table with `LowCardinality` column from `Values` format. Fixes [#21140](https://github.com/ClickHouse/ClickHouse/issues/21140). [#21357](https://github.com/ClickHouse/ClickHouse/pull/21357) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix SIGSEGV for distributed queries on failures. [#21434](https://github.com/ClickHouse/ClickHouse/pull/21434) ([Azat Khuzhin](https://github.com/azat)). +* Fix a deadlock in `ALTER DELETE` mutations for non replicated MergeTree table engines when the predicate contains the table itself. Fixes [#20558](https://github.com/ClickHouse/ClickHouse/issues/20558). [#21477](https://github.com/ClickHouse/ClickHouse/pull/21477) ([alesapin](https://github.com/alesapin)). + +#### Build/Testing/Packaging Improvement +* Fixed port clash from test_storage_kerberized_hdfs test. [#19974](https://github.com/ClickHouse/ClickHouse/pull/19974) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix some of the issues found by Coverity. See [#19964](https://github.com/ClickHouse/ClickHouse/issues/19964). [#20010](https://github.com/ClickHouse/ClickHouse/pull/20010) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to build ClickHouse with AVX-2 enabled globally. It gives slight performance benefits on modern CPUs. Not recommended for production and will not be supported as official build for now. [#20180](https://github.com/ClickHouse/ClickHouse/pull/20180) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add some checks in order by fix the bug https://clickhouse-test-reports.s3.yandex.net/20472/5bdc57004682a5e0236ec630546d20ad752c2fde/stress_test_(thread)/stderr.log. [#20516](https://github.com/ClickHouse/ClickHouse/pull/20516) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Print `stdout` and `stderr` to log when failed to start docker in integration tests. Before this PR there was a very short error message in this case which didn't help to investigate the problems:. [#20631](https://github.com/ClickHouse/ClickHouse/pull/20631) ([Vitaly Baranov](https://github.com/vitlibar)). +* Remove most of `sleep` commands from kafka integration tests, waiting for log events instead. Add test for different compression methods. [#21111](https://github.com/ClickHouse/ClickHouse/pull/21111) ([filimonov](https://github.com/filimonov)). +* Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Experimental feature + +* Introduce experimental support for window functions, enabled with `allow_experimental_functions = 1`. This is a preliminary, alpha-quality implementation that is not suitable for production use and will change in backward-incompatible ways in future releases. Please see [the documentation](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/sql-reference/window-functions/index.md#experimental-window-functions) for the list of supported features. [#20337](https://github.com/ClickHouse/ClickHouse/pull/20337) ([Alexander Kuzmenkov](https://github.com/akuzm)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Fix access control manager destruction order"'. [#20394](https://github.com/ClickHouse/ClickHouse/pull/20394) ([alesapin](https://github.com/alesapin)). +* NO CL ENTRY: 'Update argmax.md '. [#20625](https://github.com/ClickHouse/ClickHouse/pull/20625) ([Marvin Taschenberger](https://github.com/Taschenbergerm)). + diff --git a/docs/changelogs/v21.3.10.1-lts.md b/docs/changelogs/v21.3.10.1-lts.md new file mode 100644 index 00000000000..49ece009ad1 --- /dev/null +++ b/docs/changelogs/v21.3.10.1-lts.md @@ -0,0 +1,8 @@ +### ClickHouse release v21.3.10.1-lts FIXME as compared to v21.3.9.83-lts + +#### Bug Fix +* Backported in [#23973](https://github.com/ClickHouse/ClickHouse/issues/23973): Fixed a bug in recovery of staled `ReplicatedMergeTree` replica. Some metadata updates could be ignored by staled replica if `ALTER` query was executed during downtime of the replica. [#23742](https://github.com/ClickHouse/ClickHouse/pull/23742) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23773](https://github.com/ClickHouse/ClickHouse/issues/23773): Avoid possible "Cannot schedule a task" error (in case some exception had been occurred) on INSERT into Distributed. [#23744](https://github.com/ClickHouse/ClickHouse/pull/23744) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#23817](https://github.com/ClickHouse/ClickHouse/issues/23817): Fix crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#23814](https://github.com/ClickHouse/ClickHouse/issues/23814): Fix `CLEAR COLUMN` does not work when it is referenced by materialized view. Close [#23764](https://github.com/ClickHouse/ClickHouse/issues/23764). [#23781](https://github.com/ClickHouse/ClickHouse/pull/23781) ([flynn](https://github.com/ucasfl)). + diff --git a/docs/changelogs/v21.3.11.5-lts.md b/docs/changelogs/v21.3.11.5-lts.md new file mode 100644 index 00000000000..61aa8a54688 --- /dev/null +++ b/docs/changelogs/v21.3.11.5-lts.md @@ -0,0 +1,9 @@ +### ClickHouse release v21.3.11.5-lts FIXME as compared to v21.3.10.1-lts + +#### Improvement +* Backported in [#24085](https://github.com/ClickHouse/ClickHouse/issues/24085): Support specifying table schema for postgresql dictionary source. Closes [#23958](https://github.com/ClickHouse/ClickHouse/issues/23958). [#23980](https://github.com/ClickHouse/ClickHouse/pull/23980) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Bug Fix +* Backported in [#24002](https://github.com/ClickHouse/ClickHouse/issues/24002): Fix SIGSEGV for external GROUP BY and overflow row (i.e. queries like `SELECT FROM GROUP BY WITH TOTALS SETTINGS max_bytes_before_external_group_by>0, max_rows_to_group_by>0, group_by_overflow_mode='any', totals_mode='before_having'`). [#23962](https://github.com/ClickHouse/ClickHouse/pull/23962) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#24032](https://github.com/ClickHouse/ClickHouse/issues/24032): Fix crash in MergeJoin, close [#24010](https://github.com/ClickHouse/ClickHouse/issues/24010). [#24013](https://github.com/ClickHouse/ClickHouse/pull/24013) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.3.12.2-lts.md b/docs/changelogs/v21.3.12.2-lts.md new file mode 100644 index 00000000000..cfddf8c9cdd --- /dev/null +++ b/docs/changelogs/v21.3.12.2-lts.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.3.12.2-lts FIXME as compared to v21.3.11.5-lts + +#### Bug Fix +* Backported in [#24189](https://github.com/ClickHouse/ClickHouse/issues/24189): Some `ALTER PARTITION` queries might cause `Part A intersects previous part B` and `Unexpected merged part C intersecting drop range D` errors in replication queue. It's fixed. Fixes [#23296](https://github.com/ClickHouse/ClickHouse/issues/23296). [#23997](https://github.com/ClickHouse/ClickHouse/pull/23997) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24306](https://github.com/ClickHouse/ClickHouse/issues/24306): Fixed using const `DateTime` value vs `DateTime64` column in WHERE. ... [#24100](https://github.com/ClickHouse/ClickHouse/pull/24100) ([Vasily Nemkov](https://github.com/Enmk)). +* Backported in [#24140](https://github.com/ClickHouse/ClickHouse/issues/24140): Bug: explain pipeline with` select xxx final `shows wrong pipeline: ``` dell123 :) explain pipeline select z from prewhere_move_select_final final;. [#24116](https://github.com/ClickHouse/ClickHouse/pull/24116) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#24190](https://github.com/ClickHouse/ClickHouse/issues/24190): Fix a rare bug that could lead to a partially initialized table that can serve write requests (insert/alter/so on). Now such tables will be in readonly mode. [#24122](https://github.com/ClickHouse/ClickHouse/pull/24122) ([alesapin](https://github.com/alesapin)). +* Backported in [#24235](https://github.com/ClickHouse/ClickHouse/issues/24235): Fix race condition which could happen in RBAC under a heavy load. This PR fixes [#24090](https://github.com/ClickHouse/ClickHouse/issues/24090), [#24134](https://github.com/ClickHouse/ClickHouse/issues/24134),. [#24176](https://github.com/ClickHouse/ClickHouse/pull/24176) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#24244](https://github.com/ClickHouse/ClickHouse/issues/24244): Fix abnormal server termination due to hdfs becoming not accessible during query execution. Closes [#24117](https://github.com/ClickHouse/ClickHouse/issues/24117). [#24191](https://github.com/ClickHouse/ClickHouse/pull/24191) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#24241](https://github.com/ClickHouse/ClickHouse/issues/24241): Fix wrong typo at StorageMemory, this bug was introduced at [#15127](https://github.com/ClickHouse/ClickHouse/issues/15127), now fixed, Closes [#24192](https://github.com/ClickHouse/ClickHouse/issues/24192). [#24193](https://github.com/ClickHouse/ClickHouse/pull/24193) ([张中南](https://github.com/plugine)). +* Backported in [#24353](https://github.com/ClickHouse/ClickHouse/issues/24353): Fixed a bug in moving Materialized View from Ordinary to Atomic database (`RENAME TABLE` query). Now inner table is moved to new database together with Materialized View. Fixes [#23926](https://github.com/ClickHouse/ClickHouse/issues/23926). [#24309](https://github.com/ClickHouse/ClickHouse/pull/24309) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.3.13.9-lts.md b/docs/changelogs/v21.3.13.9-lts.md new file mode 100644 index 00000000000..dccb9c53162 --- /dev/null +++ b/docs/changelogs/v21.3.13.9-lts.md @@ -0,0 +1,43 @@ +### ClickHouse release v21.3.13.9-lts FIXME as compared to v21.3.12.2-lts + +#### Improvement +* Backported in [#24794](https://github.com/ClickHouse/ClickHouse/issues/24794): Avoid hiding errors like `Limit for rows or bytes to read exceeded` for scalar subqueries. [#24545](https://github.com/ClickHouse/ClickHouse/pull/24545) ([nvartolomei](https://github.com/nvartolomei)). + +#### Bug Fix +* Backported in [#25567](https://github.com/ClickHouse/ClickHouse/issues/25567): Kafka storage may support parquet format messages. [#23412](https://github.com/ClickHouse/ClickHouse/pull/23412) ([Chao Ma](https://github.com/godliness)). +* Backported in [#24554](https://github.com/ClickHouse/ClickHouse/issues/24554): Fixed server fault when inserting data through HTTP caused an exception. This fixes [#23512](https://github.com/ClickHouse/ClickHouse/issues/23512). [#23643](https://github.com/ClickHouse/ClickHouse/pull/23643) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#23833](https://github.com/ClickHouse/ClickHouse/issues/23833): Fix error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` and `use_hedged_requests`. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#24703](https://github.com/ClickHouse/ClickHouse/issues/24703): Allow empty HTTP headers. Fixes [#23901](https://github.com/ClickHouse/ClickHouse/issues/23901). [#24285](https://github.com/ClickHouse/ClickHouse/pull/24285) ([Ivan](https://github.com/abyss7)). +* Backported in [#24702](https://github.com/ClickHouse/ClickHouse/issues/24702): Fix drop partition with intersect fake parts. In rare cases there might be parts with mutation version greater than current block number. [#24321](https://github.com/ClickHouse/ClickHouse/pull/24321) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#24725](https://github.com/ClickHouse/ClickHouse/issues/24725): In "multipart/form-data" message consider the CRLF preceding a boundary as part of it. Fixes [#23905](https://github.com/ClickHouse/ClickHouse/issues/23905). [#24399](https://github.com/ClickHouse/ClickHouse/pull/24399) ([Ivan](https://github.com/abyss7)). +* Backported in [#24826](https://github.com/ClickHouse/ClickHouse/issues/24826): - Fixed the deadlock that can happen during LDAP role (re)mapping, when LDAP group is mapped to a nonexistent local role. [#24431](https://github.com/ClickHouse/ClickHouse/pull/24431) ([Denis Glazachev](https://github.com/traceon)). +* Backported in [#25566](https://github.com/ClickHouse/ClickHouse/issues/25566): Fix incorrect monotonicity of toWeek function. This fixes [#24422](https://github.com/ClickHouse/ClickHouse/issues/24422) . This bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/5212 , and was exposed later by smarter partition pruner. [#24446](https://github.com/ClickHouse/ClickHouse/pull/24446) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#24700](https://github.com/ClickHouse/ClickHouse/issues/24700): Fixed the behavior when query `SYSTEM RESTART REPLICA` or `SYSTEM SYNC REPLICA` is being processed infinitely. This was detected on server with extremely little amount of RAM. [#24457](https://github.com/ClickHouse/ClickHouse/pull/24457) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#24698](https://github.com/ClickHouse/ClickHouse/issues/24698): Enable reading of subcolumns for distributed tables. [#24472](https://github.com/ClickHouse/ClickHouse/pull/24472) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed remote JDBC bridge timeout connection issue. Closes [#9609](https://github.com/ClickHouse/ClickHouse/issues/9609). [#24588](https://github.com/ClickHouse/ClickHouse/pull/24588) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#24771](https://github.com/ClickHouse/ClickHouse/issues/24771): Fix bug which can lead to ZooKeeper client hung inside clickhouse-server. [#24721](https://github.com/ClickHouse/ClickHouse/pull/24721) ([alesapin](https://github.com/alesapin)). +* Backported in [#24934](https://github.com/ClickHouse/ClickHouse/issues/24934): - If ZooKeeper connection was lost and replica was cloned after restoring the connection, its replication queue might contain outdated entries. It's fixed. - Fixed crash when replication queue contains intersecting virtual parts. It may rarely happen if some data part was lost. Print error in log instead of terminating. [#24777](https://github.com/ClickHouse/ClickHouse/pull/24777) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24850](https://github.com/ClickHouse/ClickHouse/issues/24850): Fix bug when exception `Mutation was killed` can be thrown to the client on mutation wait when mutation not loaded into memory yet. [#24809](https://github.com/ClickHouse/ClickHouse/pull/24809) ([alesapin](https://github.com/alesapin)). +* Backported in [#24917](https://github.com/ClickHouse/ClickHouse/issues/24917): Fix extremely rare bug on low-memory servers which can lead to the inability to perform merges without restart. Possibly fixes [#24603](https://github.com/ClickHouse/ClickHouse/issues/24603). [#24872](https://github.com/ClickHouse/ClickHouse/pull/24872) ([alesapin](https://github.com/alesapin)). +* Backported in [#25185](https://github.com/ClickHouse/ClickHouse/issues/25185): Fixed bug with declaring S3 disk at root of bucket. Earlier, it reported an error: ``` [heather] 2021.05.10 02:11:11.932234 [ 72790 ] {2ff80b7b-ec53-41cb-ac35-19bb390e1759} executeQuery: Code: 36, e.displayText() = DB::Exception: Key name is empty in path style S3 URI: (http://172.17.0.2/bucket/) (version 21.6.1.1) (from 127.0.0.1:47994) (in query: SELECT policy_name FROM system.storage_policies), Stack trace (when copying this message, always include the lines below):. [#24898](https://github.com/ClickHouse/ClickHouse/pull/24898) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#24949](https://github.com/ClickHouse/ClickHouse/issues/24949): Fix possible heap-buffer-overflow in Arrow. [#24922](https://github.com/ClickHouse/ClickHouse/pull/24922) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#25562](https://github.com/ClickHouse/ClickHouse/issues/25562): Fix extremely rare error `Tagging already tagged part` in replication queue during concurrent `alter move/replace partition`. Possibly fixes [#22142](https://github.com/ClickHouse/ClickHouse/issues/22142). [#24961](https://github.com/ClickHouse/ClickHouse/pull/24961) ([alesapin](https://github.com/alesapin)). +* Backported in [#25367](https://github.com/ClickHouse/ClickHouse/issues/25367): Fix serialization of splitted nested messages in Protobuf format. This PR fixes [#24647](https://github.com/ClickHouse/ClickHouse/issues/24647). [#25000](https://github.com/ClickHouse/ClickHouse/pull/25000) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#25561](https://github.com/ClickHouse/ClickHouse/issues/25561): Fix bug which allows creating tables with columns referencing themselves like `a UInt32 ALIAS a + 1` or `b UInt32 MATERIALIZED b`. Fixes [#24910](https://github.com/ClickHouse/ClickHouse/issues/24910), [#24292](https://github.com/ClickHouse/ClickHouse/issues/24292). [#25059](https://github.com/ClickHouse/ClickHouse/pull/25059) ([alesapin](https://github.com/alesapin)). +* Backported in [#25105](https://github.com/ClickHouse/ClickHouse/issues/25105): Fix bug with constant maps in mapContains that lead to error `empty column was returned by function mapContains`. Closes [#25077](https://github.com/ClickHouse/ClickHouse/issues/25077). [#25080](https://github.com/ClickHouse/ClickHouse/pull/25080) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#25142](https://github.com/ClickHouse/ClickHouse/issues/25142): Fix crash in query with cross join and `joined_subquery_requires_alias = 0`. Fixes [#24011](https://github.com/ClickHouse/ClickHouse/issues/24011). [#25082](https://github.com/ClickHouse/ClickHouse/pull/25082) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25351](https://github.com/ClickHouse/ClickHouse/issues/25351): Fix TOCTOU error in installation script. [#25277](https://github.com/ClickHouse/ClickHouse/pull/25277) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#25473](https://github.com/ClickHouse/ClickHouse/issues/25473): Fix joinGetOrNull with not-nullable columns. This fixes [#24261](https://github.com/ClickHouse/ClickHouse/issues/24261). [#25288](https://github.com/ClickHouse/ClickHouse/pull/25288) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#25361](https://github.com/ClickHouse/ClickHouse/issues/25361): Fix error `Bad cast from type DB::ColumnLowCardinality to DB::ColumnVector` for queries where `LowCardinality` argument was used for IN (this bug appeared in 21.6). Fixes [#25187](https://github.com/ClickHouse/ClickHouse/issues/25187). [#25290](https://github.com/ClickHouse/ClickHouse/pull/25290) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25355](https://github.com/ClickHouse/ClickHouse/issues/25355): Fix Logical Error Cannot sum Array/Tuple in min/maxMap. [#25298](https://github.com/ClickHouse/ClickHouse/pull/25298) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#25437](https://github.com/ClickHouse/ClickHouse/issues/25437): Support `SimpleAggregateFunction(LowCardinality)` for `SummingMergeTree`. Fixes [#25134](https://github.com/ClickHouse/ClickHouse/issues/25134). [#25300](https://github.com/ClickHouse/ClickHouse/pull/25300) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25364](https://github.com/ClickHouse/ClickHouse/issues/25364): On ZooKeeper connection loss `ReplicatedMergeTree` table might wait for background operations to complete before trying to reconnect. It's fixed, now background operations are stopped forcefully. [#25306](https://github.com/ClickHouse/ClickHouse/pull/25306) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25387](https://github.com/ClickHouse/ClickHouse/issues/25387): Fix the possibility of non-deterministic behaviour of the `quantileDeterministic` function and similar. This closes [#20480](https://github.com/ClickHouse/ClickHouse/issues/20480). [#25313](https://github.com/ClickHouse/ClickHouse/pull/25313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#25455](https://github.com/ClickHouse/ClickHouse/issues/25455): Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25406](https://github.com/ClickHouse/ClickHouse/issues/25406): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#25505](https://github.com/ClickHouse/ClickHouse/issues/25505): Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Backport [#24721](https://github.com/ClickHouse/ClickHouse/issues/24721) to 21.3: Remove endless `wait` from ZooKeeper client"'. [#24799](https://github.com/ClickHouse/ClickHouse/pull/24799) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.3.14.1-lts.md b/docs/changelogs/v21.3.14.1-lts.md new file mode 100644 index 00000000000..b2408471ccd --- /dev/null +++ b/docs/changelogs/v21.3.14.1-lts.md @@ -0,0 +1,10 @@ +### ClickHouse release v21.3.14.1-lts FIXME as compared to v21.3.13.9-lts + +#### Bug Fix +* Backported in [#25851](https://github.com/ClickHouse/ClickHouse/issues/25851): `CAST` from `Date` to `DateTime` (or `DateTime64`) was not using the timezone of the `DateTime` type. It can also affect the comparison between `Date` and `DateTime`. Inference of the common type for `Date` and `DateTime` also was not using the corresponding timezone. It affected the results of function `if` and array construction. Closes [#24128](https://github.com/ClickHouse/ClickHouse/issues/24128). [#24129](https://github.com/ClickHouse/ClickHouse/pull/24129) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#25678](https://github.com/ClickHouse/ClickHouse/issues/25678): Fixed bug in deserialization of random generator state with might cause some data types such as `AggregateFunction(groupArraySample(N), T))` to behave in a non-deterministic way. [#24538](https://github.com/ClickHouse/ClickHouse/pull/24538) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25637](https://github.com/ClickHouse/ClickHouse/issues/25637): Fix wrong totals for query `WITH TOTALS` and `WITH FILL`. Fixes [#20872](https://github.com/ClickHouse/ClickHouse/issues/20872). [#25539](https://github.com/ClickHouse/ClickHouse/pull/25539) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#25651](https://github.com/ClickHouse/ClickHouse/issues/25651): Fix null pointer dereference in `EXPLAIN AST` without query. [#25631](https://github.com/ClickHouse/ClickHouse/pull/25631) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25716](https://github.com/ClickHouse/ClickHouse/issues/25716): `REPLACE PARTITION` might be ignored in rare cases if the source partition was empty. It's fixed. Fixes [#24869](https://github.com/ClickHouse/ClickHouse/issues/24869). [#25665](https://github.com/ClickHouse/ClickHouse/pull/25665) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25712](https://github.com/ClickHouse/ClickHouse/issues/25712): Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.3.15.4-stable.md b/docs/changelogs/v21.3.15.4-stable.md new file mode 100644 index 00000000000..c6b7a15aa4d --- /dev/null +++ b/docs/changelogs/v21.3.15.4-stable.md @@ -0,0 +1,6 @@ +### ClickHouse release v21.3.15.4-stable FIXME as compared to v21.3.14.1-lts + +#### Bug Fix +* Backported in [#25956](https://github.com/ClickHouse/ClickHouse/issues/25956): Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). +* Backported in [#26141](https://github.com/ClickHouse/ClickHouse/issues/26141): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.3.16.5-lts.md b/docs/changelogs/v21.3.16.5-lts.md new file mode 100644 index 00000000000..b5b31b64488 --- /dev/null +++ b/docs/changelogs/v21.3.16.5-lts.md @@ -0,0 +1,25 @@ +### ClickHouse release v21.3.16.5-lts FIXME as compared to v21.3.15.4-stable + +#### Bug Fix +* Backported in [#26940](https://github.com/ClickHouse/ClickHouse/issues/26940): Do not remove data on ReplicatedMergeTree table shutdown to avoid creating data to metadata inconsistency. [#26716](https://github.com/ClickHouse/ClickHouse/pull/26716) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#26983](https://github.com/ClickHouse/ClickHouse/issues/26983): Aggregate function parameters might be lost when applying some combinators causing exceptions like `Conversion from AggregateFunction(topKArray, Array(String)) to AggregateFunction(topKArray(10), Array(String)) is not supported`. It's fixed. Fixes [#26196](https://github.com/ClickHouse/ClickHouse/issues/26196) and [#26433](https://github.com/ClickHouse/ClickHouse/issues/26433). [#26814](https://github.com/ClickHouse/ClickHouse/pull/26814) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#26998](https://github.com/ClickHouse/ClickHouse/issues/26998): Fix reading of custom TLDs (stops processing with lower buffer or bigger file). [#26948](https://github.com/ClickHouse/ClickHouse/pull/26948) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27088](https://github.com/ClickHouse/ClickHouse/issues/27088): Now partition ID in queries like `ALTER TABLE ... PARTITION ID xxx` validates for correctness. Fixes [#25718](https://github.com/ClickHouse/ClickHouse/issues/25718). [#26963](https://github.com/ClickHouse/ClickHouse/pull/26963) ([alesapin](https://github.com/alesapin)). +* Backported in [#27049](https://github.com/ClickHouse/ClickHouse/issues/27049): [RFC] Fix possible mutation stack due to race with DROP_RANGE. [#27002](https://github.com/ClickHouse/ClickHouse/pull/27002) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27158](https://github.com/ClickHouse/ClickHouse/issues/27158): Fix synchronization in GRPCServer This PR fixes [#27024](https://github.com/ClickHouse/ClickHouse/issues/27024). [#27064](https://github.com/ClickHouse/ClickHouse/pull/27064) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#27368](https://github.com/ClickHouse/ClickHouse/issues/27368): - Fix uninitialized memory in functions `multiSearch*` with empty array, close [#27169](https://github.com/ClickHouse/ClickHouse/issues/27169). [#27181](https://github.com/ClickHouse/ClickHouse/pull/27181) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#27264](https://github.com/ClickHouse/ClickHouse/issues/27264): In rare cases `system.detached_parts` table might contain incorrect information for some parts, it's fixed. Fixes [#27114](https://github.com/ClickHouse/ClickHouse/issues/27114). [#27183](https://github.com/ClickHouse/ClickHouse/pull/27183) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#27413](https://github.com/ClickHouse/ClickHouse/issues/27413): Fixed incorrect validation of partition id for MergeTree tables that created with old syntax. [#27328](https://github.com/ClickHouse/ClickHouse/pull/27328) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#27645](https://github.com/ClickHouse/ClickHouse/issues/27645): Fix incorrect result for query with row-level security, prewhere and LowCardinality filter. Fixes [#27179](https://github.com/ClickHouse/ClickHouse/issues/27179). [#27329](https://github.com/ClickHouse/ClickHouse/pull/27329) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#27473](https://github.com/ClickHouse/ClickHouse/issues/27473): fix metric BackgroundMessageBrokerSchedulePoolTask, maybe mistyped。. [#27452](https://github.com/ClickHouse/ClickHouse/pull/27452) ([Ben](https://github.com/benbiti)). +* Backported in [#27864](https://github.com/ClickHouse/ClickHouse/issues/27864): Prevent crashes for some formats when NULL (tombstone) message was coming from Kafka. Closes [#19255](https://github.com/ClickHouse/ClickHouse/issues/19255). [#27794](https://github.com/ClickHouse/ClickHouse/pull/27794) ([filimonov](https://github.com/filimonov)). +* Backported in [#28207](https://github.com/ClickHouse/ClickHouse/issues/28207): Fix cases, when read buffer fails with 'attempt to read after end of file'. Closes [#26149](https://github.com/ClickHouse/ClickHouse/issues/26149). [#28150](https://github.com/ClickHouse/ClickHouse/pull/28150) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + +#### Build/Testing/Packaging Improvement +* Backported in [#28075](https://github.com/ClickHouse/ClickHouse/issues/28075): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#28181](https://github.com/ClickHouse/ClickHouse/issues/28181): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). +* Backported in [#28293](https://github.com/ClickHouse/ClickHouse/issues/28293): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.3.17.2-lts.md b/docs/changelogs/v21.3.17.2-lts.md new file mode 100644 index 00000000000..b6b48ddab61 --- /dev/null +++ b/docs/changelogs/v21.3.17.2-lts.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.3.17.2-lts FIXME as compared to v21.3.16.5-lts + +#### Bug Fix +* Backported in [#28647](https://github.com/ClickHouse/ClickHouse/issues/28647): Fix a rare bug in `DROP PART` which can lead to the error `Unexpected merged part intersects drop range`. [#27807](https://github.com/ClickHouse/ClickHouse/pull/27807) ([alesapin](https://github.com/alesapin)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#28569](https://github.com/ClickHouse/ClickHouse/issues/28569): Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)). +* Backported in [#28857](https://github.com/ClickHouse/ClickHouse/issues/28857): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)). +* Backported in [#28995](https://github.com/ClickHouse/ClickHouse/issues/28995): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28929](https://github.com/ClickHouse/ClickHouse/issues/28929): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.3.18.4-lts.md b/docs/changelogs/v21.3.18.4-lts.md new file mode 100644 index 00000000000..612b3660e3b --- /dev/null +++ b/docs/changelogs/v21.3.18.4-lts.md @@ -0,0 +1,22 @@ +### ClickHouse release v21.3.18.4-lts FIXME as compared to v21.3.17.2-lts + +#### Improvement +* Backported in [#30355](https://github.com/ClickHouse/ClickHouse/issues/30355): Use separate `clickhouse-bridge` group and user for bridge processes. Set oom_score_adj so the bridges will be first subjects for OOM killer. Set set maximum RSS to 1 GiB. Closes [#23861](https://github.com/ClickHouse/ClickHouse/issues/23861). [#25280](https://github.com/ClickHouse/ClickHouse/pull/25280) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29945](https://github.com/ClickHouse/ClickHouse/issues/29945): Update zoneinfo files to 2021c. [#29925](https://github.com/ClickHouse/ClickHouse/pull/29925) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#30041](https://github.com/ClickHouse/ClickHouse/issues/30041): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#29260](https://github.com/ClickHouse/ClickHouse/issues/29260): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#29026](https://github.com/ClickHouse/ClickHouse/issues/29026): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29194](https://github.com/ClickHouse/ClickHouse/issues/29194): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#29360](https://github.com/ClickHouse/ClickHouse/issues/29360): Fix possible `Table columns structure in ZooKeeper is different from local table structure` exception while recreating or creating new replicas of `ReplicatedMergeTree`, when one of table columns have default expressions with case-insensitive functions. [#29266](https://github.com/ClickHouse/ClickHouse/pull/29266) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#29300](https://github.com/ClickHouse/ClickHouse/issues/29300): Fix connection timeouts (`send_timeout`/`receive_timeout`). [#29282](https://github.com/ClickHouse/ClickHouse/pull/29282) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30042](https://github.com/ClickHouse/ClickHouse/issues/30042): Condition in filter predicate could be lost after push-down optimisation. [#29625](https://github.com/ClickHouse/ClickHouse/pull/29625) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29848](https://github.com/ClickHouse/ClickHouse/issues/29848): Fix concurrent access to `LowCardinality` during `GROUP BY` (leads to SIGSEGV). [#29782](https://github.com/ClickHouse/ClickHouse/pull/29782) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30066](https://github.com/ClickHouse/ClickHouse/issues/30066): Fix crash of sample by `tuple()`, closes [#30004](https://github.com/ClickHouse/ClickHouse/issues/30004). [#30016](https://github.com/ClickHouse/ClickHouse/pull/30016) ([flynn](https://github.com/ucasfl)). +* Backported in [#30332](https://github.com/ClickHouse/ClickHouse/issues/30332): * Allow identifiers staring with numbers in multiple joins. [#30230](https://github.com/ClickHouse/ClickHouse/pull/30230) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#30379](https://github.com/ClickHouse/ClickHouse/issues/30379): fix replaceRegexpAll bug. [#30292](https://github.com/ClickHouse/ClickHouse/pull/30292) ([Memo](https://github.com/Joeywzr)). + diff --git a/docs/changelogs/v21.3.19.1-lts.md b/docs/changelogs/v21.3.19.1-lts.md new file mode 100644 index 00000000000..6a2be8d6dcb --- /dev/null +++ b/docs/changelogs/v21.3.19.1-lts.md @@ -0,0 +1,26 @@ +### ClickHouse release v21.3.19.1-lts FIXME as compared to v21.3.18.4-lts + +#### Performance Improvement +* Backported in [#31733](https://github.com/ClickHouse/ClickHouse/issues/31733): Improve performance of JSON and XML output formats. [#31673](https://github.com/ClickHouse/ClickHouse/pull/31673) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#31577](https://github.com/ClickHouse/ClickHouse/issues/31577): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). +* Backported in [#32347](https://github.com/ClickHouse/ClickHouse/issues/32347): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release + +* Backported in [#30913](https://github.com/ClickHouse/ClickHouse/issues/30913): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#30750](https://github.com/ClickHouse/ClickHouse/issues/30750): Functions for case-insensitive search in UTF8 strings like `positionCaseInsensitiveUTF8` and `countSubstringsCaseInsensitiveUTF8` might find substrings that actually does not match, it's fixed. [#30663](https://github.com/ClickHouse/ClickHouse/pull/30663) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31038](https://github.com/ClickHouse/ClickHouse/issues/31038): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31519](https://github.com/ClickHouse/ClickHouse/issues/31519): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)). +* Backported in [#31581](https://github.com/ClickHouse/ClickHouse/issues/31581): * Disable `partial_merge_join_left_table_buffer_bytes` before bug in this optimization is fixed. See [#31009](https://github.com/ClickHouse/ClickHouse/issues/31009)). * Remove redundant option `partial_merge_join_optimizations`. [#31528](https://github.com/ClickHouse/ClickHouse/pull/31528) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#31792](https://github.com/ClickHouse/ClickHouse/issues/31792): Settings `input_format_allow_errors_num` and `input_format_allow_errors_ratio` did not work for parsing of domain types, such as `IPv4`, it's fixed. Fixes [#31686](https://github.com/ClickHouse/ClickHouse/issues/31686). [#31697](https://github.com/ClickHouse/ClickHouse/pull/31697) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31816](https://github.com/ClickHouse/ClickHouse/issues/31816): Fix race in JSONEachRowWithProgress output format when data and lines with progress are mixed in output. [#31736](https://github.com/ClickHouse/ClickHouse/pull/31736) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31758](https://github.com/ClickHouse/ClickHouse/issues/31758): Fix usage of `Buffer` table engine with type `Map`. Fixes [#30546](https://github.com/ClickHouse/ClickHouse/issues/30546). [#31742](https://github.com/ClickHouse/ClickHouse/pull/31742) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#32153](https://github.com/ClickHouse/ClickHouse/issues/32153): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#31894](https://github.com/ClickHouse/ClickHouse/issues/31894): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#32214](https://github.com/ClickHouse/ClickHouse/issues/32214): Number of active replicas might be determined incorrectly when inserting with quorum if setting `replicated_can_become_leader` is disabled on some replicas. It's fixed. [#32157](https://github.com/ClickHouse/ClickHouse/pull/32157) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.3.2.5-lts.md b/docs/changelogs/v21.3.2.5-lts.md new file mode 100644 index 00000000000..5135f909b7f --- /dev/null +++ b/docs/changelogs/v21.3.2.5-lts.md @@ -0,0 +1,161 @@ +### ClickHouse release v21.3.2.5-lts FIXME as compared to v21.2.1.5869-prestable + +#### Backward Incompatible Change +* Now all case-insensitive function names will be lower-cased during query analysis. This is needed for projection query routing. [#20174](https://github.com/ClickHouse/ClickHouse/pull/20174) ([Amos Bird](https://github.com/amosbird)). +* Now it's not allowed to create MergeTree tables in old syntax with table TTL because it's just ignored. Attach of old tables is still possible. [#20282](https://github.com/ClickHouse/ClickHouse/pull/20282) ([alesapin](https://github.com/alesapin)). + +#### New Feature +* Add experimental `Replicated` database engine. It replicates DDL queries across multiple hosts. [#16193](https://github.com/ClickHouse/ClickHouse/pull/16193) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Distributed query deduplication is a followup to [#16033](https://github.com/ClickHouse/ClickHouse/issues/16033) and partially resolves the proposal [#13574](https://github.com/ClickHouse/ClickHouse/issues/13574). [#17348](https://github.com/ClickHouse/ClickHouse/pull/17348) ([xjewer](https://github.com/xjewer)). +* Add the ability to backup/restore metadata files for DiskS3. [#18377](https://github.com/ClickHouse/ClickHouse/pull/18377) ([Pavel Kovalenko](https://github.com/Jokser)). +* - Included in the pull request. [#18508](https://github.com/ClickHouse/ClickHouse/pull/18508) ([PHO](https://github.com/depressed-pho)). +* Added file() function to read file as a String. This close [#issue:18851](https://github.com/ClickHouse/ClickHouse/issues/18851). [#19204](https://github.com/ClickHouse/ClickHouse/pull/19204) ([keenwolf](https://github.com/keen-wolf)). +* Tables with `MergeTree*` engine now have two new table-level settings for query concurrency control. Setting `max_concurrent_queries` limits the number of concurrently executed queries which are related to this table. Setting `min_marks_to_honor_max_concurrent_queries` tells to apply previous setting only if query reads at least this number of marks. [#19544](https://github.com/ClickHouse/ClickHouse/pull/19544) ([Amos Bird](https://github.com/amosbird)). +* - Mentioned in [#18454](https://github.com/ClickHouse/ClickHouse/issues/18454) - add function `htmlOrxmlCoarseParse`; - support `` parse; - support `` parse; - support `` parse; - support white space collapse; - support any `` format parse; - HyperScan to support SIMD; - Everything is done in a single pass. [#19600](https://github.com/ClickHouse/ClickHouse/pull/19600) ([zlx19950903](https://github.com/zlx19950903)). +* Add quota type QUERY_SELECTS and QUERY_INSERTS. [#19603](https://github.com/ClickHouse/ClickHouse/pull/19603) ([JackyWoo](https://github.com/JackyWoo)). +* ExecutableDictionarySource added implicit_key option. Fixes [#14527](https://github.com/ClickHouse/ClickHouse/issues/14527). [#19677](https://github.com/ClickHouse/ClickHouse/pull/19677) ([Maksim Kita](https://github.com/kitaisreal)). +* Added Server Side Encryption Customer Keys (the `x-amz-server-side-encryption-customer-(key/md5)` header) support in S3 client. See [the link](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). Closes [#19428](https://github.com/ClickHouse/ClickHouse/issues/19428). [#19748](https://github.com/ClickHouse/ClickHouse/pull/19748) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Function `reinterpretAs` updated to support big integers. Fixes [#19691](https://github.com/ClickHouse/ClickHouse/issues/19691). [#19858](https://github.com/ClickHouse/ClickHouse/pull/19858) ([Maksim Kita](https://github.com/kitaisreal)). +* Add setting `insert_shard_id` to support insert data into specific shard from distributed table. [#19961](https://github.com/ClickHouse/ClickHouse/pull/19961) ([flynn](https://github.com/ucasfl)). +* Added timezoneOffset(datetime) function which will give the offset from UTC in seconds. This close [#issue:19850](https://github.com/ClickHouse/ClickHouse/issues/19850). [#19962](https://github.com/ClickHouse/ClickHouse/pull/19962) ([keenwolf](https://github.com/keen-wolf)). +* New `event_time_microseconds column` in `system.part_log` table. [#20027](https://github.com/ClickHouse/ClickHouse/pull/20027) ([Bharat Nallan](https://github.com/bharatnc)). +* ... Add aggregate function `deltaSum` for summing the differences between consecutive rows. [#20057](https://github.com/ClickHouse/ClickHouse/pull/20057) ([Russ Frank](https://github.com/rf)). +* Add two settings to delay or throw error during insertion when there are too many inactive parts. This is useful when server fails to clean up parts quickly enough. [#20178](https://github.com/ClickHouse/ClickHouse/pull/20178) ([Amos Bird](https://github.com/amosbird)). +* Add file engine settings: `engine_file_empty_if_not_exists` and `engine_file_truncate_on_insert`. [#20620](https://github.com/ClickHouse/ClickHouse/pull/20620) ([M0r64n](https://github.com/M0r64n)). + +#### Performance Improvement +* Add parallel select final for one part with level>0 when `do_not_merge_across_partitions_select_final` setting is 1. [#19375](https://github.com/ClickHouse/ClickHouse/pull/19375) ([Kruglov Pavel](https://github.com/Avogar)). +* Improved performance of bitmap columns during joins. [#19407](https://github.com/ClickHouse/ClickHouse/pull/19407) ([templarzq](https://github.com/templarzq)). +* Partially reimplement HTTP server to make it making less copies of incoming and outgoing data. It gives up to 1.5 performance improvement on inserting long records over HTTP. [#19516](https://github.com/ClickHouse/ClickHouse/pull/19516) ([Ivan](https://github.com/abyss7)). +* Improve performance of aggregate functions by more strict aliasing. [#19946](https://github.com/ClickHouse/ClickHouse/pull/19946) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix the case when DataType parser may have exponential complexity (found by fuzzer). This closes [#20096](https://github.com/ClickHouse/ClickHouse/issues/20096). [#20132](https://github.com/ClickHouse/ClickHouse/pull/20132) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `compress` setting for `Memory` tables. If it's enabled the table will use less RAM. On some machines and datasets it can also work faster on SELECT, but it is not always the case. This closes [#20093](https://github.com/ClickHouse/ClickHouse/issues/20093). Note: there are reasons why Memory tables can work slower than MergeTree: (1) lack of compression (2) static size of blocks (3) lack of indices and prewhere... [#20168](https://github.com/ClickHouse/ClickHouse/pull/20168) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not squash blocks too much on INSERT SELECT if inserting into Memory table. In previous versions inefficient data representation was created in Memory table after INSERT SELECT. This closes [#13052](https://github.com/ClickHouse/ClickHouse/issues/13052). [#20169](https://github.com/ClickHouse/ClickHouse/pull/20169) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improved performance of aggregation by several fixed size fields (unconfirmed). [#20454](https://github.com/ClickHouse/ClickHouse/pull/20454) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Speed up reading from `Memory` tables in extreme cases (when reading speed is in order of 50 GB/sec) by simplification of pipeline and (consequently) less lock contention in pipeline scheduling. [#20468](https://github.com/ClickHouse/ClickHouse/pull/20468) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve performance of GROUP BY multiple fixed size keys. [#20472](https://github.com/ClickHouse/ClickHouse/pull/20472) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The setting `distributed_aggregation_memory_efficient` is enabled by default. It will lower memory usage and improve performance of distributed queries. [#20599](https://github.com/ClickHouse/ClickHouse/pull/20599) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Slightly better code in aggregation. [#20978](https://github.com/ClickHouse/ClickHouse/pull/20978) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add back intDiv/module vectorConstant specializations for better performance. This fixes [#21293](https://github.com/ClickHouse/ClickHouse/issues/21293) . The regression was introduced in https://github.com/ClickHouse/ClickHouse/pull/18145 . [#21307](https://github.com/ClickHouse/ClickHouse/pull/21307) ([Amos Bird](https://github.com/amosbird)). + +#### Improvement +* Fix creation of `TTL` in cases, when its expression is a function and it is the same as `ORDER BY` key. Now it's allowed to set custom aggregation to primary key columns in `TTL` with `GROUP BY`. Backward incompatible: For primary key columns, which are not in `GROUP BY` and aren't set explicitly now is applied function `any` instead of `max`, when TTL is expired. Also if you use TTL with `WHERE` or `GROUP BY` you can see exceptions at merges, while making rolling update. [#15450](https://github.com/ClickHouse/ClickHouse/pull/15450) ([Anton Popov](https://github.com/CurtizJ)). +* Hedged Requests for remote queries. When setting `use_hedged_requests` enabled (by default), allow to establish many connections with different replicas for query. New connection is enabled in case existent connection(s) with replica(s) were not established within `hedged_connection_timeout` or no data was received within `receive_data_timeout`. Query uses the first connection which send non empty progress packet (or data packet, if `allow_changing_replica_until_first_data_packet`); other connections are cancelled. Queries with `max_parallel_replicas > 1` are supported. [#19291](https://github.com/ClickHouse/ClickHouse/pull/19291) ([Kruglov Pavel](https://github.com/Avogar)). +* Print inline frames for fatal stacktraces. [#19317](https://github.com/ClickHouse/ClickHouse/pull/19317) ([Ivan](https://github.com/abyss7)). +* Do not silently ignore write errors. [#19451](https://github.com/ClickHouse/ClickHouse/pull/19451) ([Azat Khuzhin](https://github.com/azat)). +* Added support for `PREWHERE` when tables have row-level security expressions specified. [#19576](https://github.com/ClickHouse/ClickHouse/pull/19576) ([Denis Glazachev](https://github.com/traceon)). +* Add IStoragePolicy interface. [#19608](https://github.com/ClickHouse/ClickHouse/pull/19608) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). +* Add ability to throttle INSERT into Distributed based on amount of pending bytes for async send (`bytes_to_delay_insert`/`max_delay_to_insert` and `bytes_to_throw_insert` settings for `Distributed` engine has been added). [#19673](https://github.com/ClickHouse/ClickHouse/pull/19673) ([Azat Khuzhin](https://github.com/azat)). +* move Conditions that are not related to JOIN to where clause. [#18720](https://github.com/ClickHouse/ClickHouse/issues/18720). [#19685](https://github.com/ClickHouse/ClickHouse/pull/19685) ([hexiaoting](https://github.com/hexiaoting)). +* Add separate config directive for Buffer profile. [#19721](https://github.com/ClickHouse/ClickHouse/pull/19721) ([Azat Khuzhin](https://github.com/azat)). +* Show MaterializeMySQL tables in `system.parts`. [#19770](https://github.com/ClickHouse/ClickHouse/pull/19770) ([Stig Bakken](https://github.com/stigsb)). +* Initialize MaxDDLEntryID to the last value after restarting. Before this PR, MaxDDLEntryID will remain zero until a new DDLTask is processed. [#19924](https://github.com/ClickHouse/ClickHouse/pull/19924) ([Amos Bird](https://github.com/amosbird)). +* Add conversion of block structure for INSERT into Distributed tables if it does not match. [#19947](https://github.com/ClickHouse/ClickHouse/pull/19947) ([Azat Khuzhin](https://github.com/azat)). +* If user calls `JSONExtract` function with `Float32` type requested, allow inaccurate conversion to the result type. For example the number `0.1` in JSON is double precision and is not representable in Float32, but the user still wants to get it. Previous versions return 0 for non-Nullable type and NULL for Nullable type to indicate that conversion is imprecise. The logic was 100% correct but it was surprising to users and leading to questions. This closes [#13962](https://github.com/ClickHouse/ClickHouse/issues/13962). [#19960](https://github.com/ClickHouse/ClickHouse/pull/19960) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The value of MYSQL_OPT_RECONNECT option can now be controlled by "opt_reconnect" parameter in the config section of mysql replica. [#19998](https://github.com/ClickHouse/ClickHouse/pull/19998) ([Alexander Kazakov](https://github.com/Akazz)). +* Return `DiskType` instead of `String` in IDisk::getType() as in the rest of storage interfaces. [#19999](https://github.com/ClickHouse/ClickHouse/pull/19999) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). +* Fix data race in executable dictionary that was possible only on misuse (when the script returns data ignoring its input). [#20045](https://github.com/ClickHouse/ClickHouse/pull/20045) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Show full details of MaterializeMySQL tables in `system.tables`. [#20051](https://github.com/ClickHouse/ClickHouse/pull/20051) ([Stig Bakken](https://github.com/stigsb)). +* Supports system.zookeeper path IN query. [#20105](https://github.com/ClickHouse/ClickHouse/pull/20105) ([小路](https://github.com/nicelulu)). +* 1. SHOW TABLES is now considered as one query in the quota calculations, not two queries. 2. SYSTEM queries now consume quota. 3. Fix calculation of interval's end in quota consumption. [#20106](https://github.com/ClickHouse/ClickHouse/pull/20106) ([Vitaly Baranov](https://github.com/vitlibar)). +* - Fix toDateTime64(toDate()/toDateTime()) for DateTime64 - Implement DateTime64 clamping to match DateTime behaviour. [#20131](https://github.com/ClickHouse/ClickHouse/pull/20131) ([Azat Khuzhin](https://github.com/azat)). +* The setting `access_management` is now configurable on startup by providing `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT`, defaults to disabled (`0`) which was the prior value. [#20139](https://github.com/ClickHouse/ClickHouse/pull/20139) ([Marquitos](https://github.com/sonirico)). +* Updated `CacheDictionary`, `ComplexCacheDictionary`, `SSDCacheDictionary`, `SSDComplexKeyDictionary` to use LRUHashMap as underlying index. [#20164](https://github.com/ClickHouse/ClickHouse/pull/20164) ([Maksim Kita](https://github.com/kitaisreal)). +* Support all native integer types in bitmap functions. [#20171](https://github.com/ClickHouse/ClickHouse/pull/20171) ([Amos Bird](https://github.com/amosbird)). +* Normalize count(constant), sum(1) to count(). This is needed for projection query routing. [#20175](https://github.com/ClickHouse/ClickHouse/pull/20175) ([Amos Bird](https://github.com/amosbird)). +* Perform algebraic optimizations of arithmetic expressions inside `avg` aggregate function. close [#20092](https://github.com/ClickHouse/ClickHouse/issues/20092). [#20183](https://github.com/ClickHouse/ClickHouse/pull/20183) ([flynn](https://github.com/ucasfl)). +* Lockless `SYSTEM FLUSH DISTRIBUTED`. [#20215](https://github.com/ClickHouse/ClickHouse/pull/20215) ([Azat Khuzhin](https://github.com/azat)). +* Implicit conversion from integer to Dicimal type might succeeded if integer value doe not fit into Decimal type. Now it throws `ARGUMENT_OUT_OF_BOUND`. [#20232](https://github.com/ClickHouse/ClickHouse/pull/20232) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Do not allow early constant folding of explicitly forbidden functions. [#20303](https://github.com/ClickHouse/ClickHouse/pull/20303) ([Azat Khuzhin](https://github.com/azat)). +* Make FQDN and other DNS related functions work correctly in alpine images. [#20336](https://github.com/ClickHouse/ClickHouse/pull/20336) ([filimonov](https://github.com/filimonov)). +* Fixed race between execution of distributed DDL tasks and cleanup of DDL queue. Now DDL task cannot be removed from ZooKeeper if there are active workers. Fixes [#20016](https://github.com/ClickHouse/ClickHouse/issues/20016). [#20448](https://github.com/ClickHouse/ClickHouse/pull/20448) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improved serialization for data types combined of Arrays and Tuples. Improved matching enum data types to protobuf enum type. Fixed serialization of the `Map` data type. Omitted values are now set by default. [#20506](https://github.com/ClickHouse/ClickHouse/pull/20506) ([Vitaly Baranov](https://github.com/vitlibar)). +* https://github.com/ClickHouse/ClickHouse/issues/20576. [#20596](https://github.com/ClickHouse/ClickHouse/pull/20596) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Function 'reinterpretAs(x, Type)' renamed into 'reinterpret(x, Type)'. [#20611](https://github.com/ClickHouse/ClickHouse/pull/20611) ([Maksim Kita](https://github.com/kitaisreal)). +* When loading config for mysql source ClickHouse will now randomize the list of replicas with the same priority to ensure the round-robin logics of picking mysql endpoint. This closes [#20629](https://github.com/ClickHouse/ClickHouse/issues/20629). [#20632](https://github.com/ClickHouse/ClickHouse/pull/20632) ([Alexander Kazakov](https://github.com/Akazz)). +* Do only merging of sorted blocks on initiator with distributed_group_by_no_merge. [#20882](https://github.com/ClickHouse/ClickHouse/pull/20882) ([Azat Khuzhin](https://github.com/azat)). +* - Fill only requested columns when querying system.parts & system.parts_columns. Closes [#19570](https://github.com/ClickHouse/ClickHouse/issues/19570). ... [#21035](https://github.com/ClickHouse/ClickHouse/pull/21035) ([Anmol Arora](https://github.com/anmolarora)). +* Usability improvement: more consistent `DateTime64` parsing: recognize the case when unix timestamp with subsecond resolution is specified as scaled integer (like `1111111111222` instead of `1111111111.222`). This closes [#13194](https://github.com/ClickHouse/ClickHouse/issues/13194). [#21053](https://github.com/ClickHouse/ClickHouse/pull/21053) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* MySQL dictionary source will now retry unexpected connection failures (Lost connection to MySQL server during query) which sometimes happen on SSL/TLS connections. [#21237](https://github.com/ClickHouse/ClickHouse/pull/21237) ([Alexander Kazakov](https://github.com/Akazz)). +* Forbid to drop a column if it's referenced by materialized view. Closes [#21164](https://github.com/ClickHouse/ClickHouse/issues/21164). [#21303](https://github.com/ClickHouse/ClickHouse/pull/21303) ([flynn](https://github.com/ucasfl)). +* Provide better compatibility for mysql clients. 1. mysql jdbc 2. mycli. [#21367](https://github.com/ClickHouse/ClickHouse/pull/21367) ([Amos Bird](https://github.com/amosbird)). +* Case-insensitive compression methods for table functions. Also fixed `LZMA` compression method which was checked in upper case. [#21416](https://github.com/ClickHouse/ClickHouse/pull/21416) ([Vladimir Chebotarev](https://github.com/excitoon)). + +#### Bug Fix +* Background thread which executes `ON CLUSTER` queries might hang waiting for dropped replicated table to do something. It's fixed. [#19684](https://github.com/ClickHouse/ClickHouse/pull/19684) ([yiguolei](https://github.com/yiguolei)). +* Fix a bug that moving pieces to destination table may failed in case of launching multiple clickhouse-copiers. [#19743](https://github.com/ClickHouse/ClickHouse/pull/19743) ([madianjun](https://github.com/mdianjun)). +* Fix clickhouse-client abort exception while executing only `select`. [#19790](https://github.com/ClickHouse/ClickHouse/pull/19790) ([李扬](https://github.com/taiyang-li)). +* Fix starting the server with tables having default expressions containing dictGet(). Allow getting return type of dictGet() without loading dictionary. [#19805](https://github.com/ClickHouse/ClickHouse/pull/19805) ([Vitaly Baranov](https://github.com/vitlibar)). +* Deadlock was possible if system.text_log is enabled. This fixes [#19874](https://github.com/ClickHouse/ClickHouse/issues/19874). [#19875](https://github.com/ClickHouse/ClickHouse/pull/19875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* BloomFilter index crash fix. Fixes [#19757](https://github.com/ClickHouse/ClickHouse/issues/19757). [#19884](https://github.com/ClickHouse/ClickHouse/pull/19884) ([Maksim Kita](https://github.com/kitaisreal)). +* - Fix a segfault in function `fromModifiedJulianDay` when the argument type is `Nullable(T)` for any integral types other than Int32. [#19959](https://github.com/ClickHouse/ClickHouse/pull/19959) ([PHO](https://github.com/depressed-pho)). +* `EmbeddedRocksDB` is an experimental storage. Fix the issue with lack of proper type checking. Simplified code. This closes [#19967](https://github.com/ClickHouse/ClickHouse/issues/19967). [#19972](https://github.com/ClickHouse/ClickHouse/pull/19972) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Prevent "Connection refused" in docker during initialization script execution. [#20012](https://github.com/ClickHouse/ClickHouse/pull/20012) ([filimonov](https://github.com/filimonov)). +* MaterializeMySQL: Fix replication for statements that update several tables. [#20066](https://github.com/ClickHouse/ClickHouse/pull/20066) ([Håvard Kvålen](https://github.com/havardk)). +* Fix the case when calculating modulo of division of negative number by small divisor, the resulting data type was not large enough to accomodate the negative result. This closes [#20052](https://github.com/ClickHouse/ClickHouse/issues/20052). [#20067](https://github.com/ClickHouse/ClickHouse/pull/20067) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* * Bugfix in StorageJoin. [#20079](https://github.com/ClickHouse/ClickHouse/pull/20079) ([Vladimir C](https://github.com/vdimir)). +* The `MongoDB` table engine now establishes connection only when it's going to read data. `ATTACH TABLE` won't try to connect anymore. [#20110](https://github.com/ClickHouse/ClickHouse/pull/20110) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix server crash after query with `if` function with `Tuple` type of then/else branches result. `Tuple` type must contain `Array` or another complex type. Fixes [#18356](https://github.com/ClickHouse/ClickHouse/issues/18356). [#20133](https://github.com/ClickHouse/ClickHouse/pull/20133) ([alesapin](https://github.com/alesapin)). +* fix toMinute function to handle special timezone correctly. [#20149](https://github.com/ClickHouse/ClickHouse/pull/20149) ([keenwolf](https://github.com/keen-wolf)). +* Fixes [#19314](https://github.com/ClickHouse/ClickHouse/issues/19314). [#20156](https://github.com/ClickHouse/ClickHouse/pull/20156) ([Ivan](https://github.com/abyss7)). +* Fix CTE when using in INSERT SELECT. This fixes [#20187](https://github.com/ClickHouse/ClickHouse/issues/20187), fixes [#20195](https://github.com/ClickHouse/ClickHouse/issues/20195). [#20211](https://github.com/ClickHouse/ClickHouse/pull/20211) ([Amos Bird](https://github.com/amosbird)). +* Fix rare server crash on config reload during the shutdown. Fixes [#19689](https://github.com/ClickHouse/ClickHouse/issues/19689). [#20224](https://github.com/ClickHouse/ClickHouse/pull/20224) ([alesapin](https://github.com/alesapin)). +* Fix exception during vertical merge for `MergeTree` table engines family which don't allow to perform vertical merges. Fixes [#20259](https://github.com/ClickHouse/ClickHouse/issues/20259). [#20279](https://github.com/ClickHouse/ClickHouse/pull/20279) ([alesapin](https://github.com/alesapin)). +* Fixed the behavior when in case of broken JSON we tried to read the whole file into memory which leads to exception from the allocator. Fixes [#19719](https://github.com/ClickHouse/ClickHouse/issues/19719). [#20286](https://github.com/ClickHouse/ClickHouse/pull/20286) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Restrict to `DROP` or `RENAME` version column of `*CollapsingMergeTree` and `ReplacingMergeTree` table engines. [#20300](https://github.com/ClickHouse/ClickHouse/pull/20300) ([alesapin](https://github.com/alesapin)). +* Fix too often retries of failed background tasks for `ReplicatedMergeTree` table engines family. This could lead to too verbose logging and increased CPU load. Fixes [#20203](https://github.com/ClickHouse/ClickHouse/issues/20203). [#20335](https://github.com/ClickHouse/ClickHouse/pull/20335) ([alesapin](https://github.com/alesapin)). +* Fix incorrect result of binary operations between two constant decimals of different scale. Fixes [#20283](https://github.com/ClickHouse/ClickHouse/issues/20283). [#20339](https://github.com/ClickHouse/ClickHouse/pull/20339) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix null dereference with `join_use_nulls=1`. [#20344](https://github.com/ClickHouse/ClickHouse/pull/20344) ([Azat Khuzhin](https://github.com/azat)). +* Avoid invalid dereference in RANGE_HASHED() dictionary. [#20345](https://github.com/ClickHouse/ClickHouse/pull/20345) ([Azat Khuzhin](https://github.com/azat)). +* Check if table function `view` is used in expression list and throw an error. This fixes [#20342](https://github.com/ClickHouse/ClickHouse/issues/20342). [#20350](https://github.com/ClickHouse/ClickHouse/pull/20350) ([Amos Bird](https://github.com/amosbird)). +* Fix `LOGICAL_ERROR` for `join_use_nulls=1` when JOIN contains const from SELECT. [#20461](https://github.com/ClickHouse/ClickHouse/pull/20461) ([Azat Khuzhin](https://github.com/azat)). +* Fix abnormal server termination when http client goes away. [#20464](https://github.com/ClickHouse/ClickHouse/pull/20464) ([Azat Khuzhin](https://github.com/azat)). +* Fix infinite loop when propagating WITH aliases to subqueries. This fixes [#20388](https://github.com/ClickHouse/ClickHouse/issues/20388). [#20476](https://github.com/ClickHouse/ClickHouse/pull/20476) ([Amos Bird](https://github.com/amosbird)). +* Fix function `transform` does not work properly for floating point keys. Closes [#20460](https://github.com/ClickHouse/ClickHouse/issues/20460). [#20479](https://github.com/ClickHouse/ClickHouse/pull/20479) ([flynn](https://github.com/ucasfl)). +* Add proper checks while parsing directory names for async INSERT (fixes SIGSEGV). [#20498](https://github.com/ClickHouse/ClickHouse/pull/20498) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash which could happen if unknown packet was received from remove query (was introduced in [#17868](https://github.com/ClickHouse/ClickHouse/issues/17868)). [#20547](https://github.com/ClickHouse/ClickHouse/pull/20547) ([Azat Khuzhin](https://github.com/azat)). +* Fix the number of threads for scalar subqueries and subqueries for index (after [#19007](https://github.com/ClickHouse/ClickHouse/issues/19007) single thread was always used). Fixes [#20457](https://github.com/ClickHouse/ClickHouse/issues/20457), [#20512](https://github.com/ClickHouse/ClickHouse/issues/20512). [#20550](https://github.com/ClickHouse/ClickHouse/pull/20550) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed inconsistent behavior of dictionary in case of queries where we look for absent keys in dictionary. [#20578](https://github.com/ClickHouse/ClickHouse/pull/20578) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix subquery with union distinct and limit clause. close [#20597](https://github.com/ClickHouse/ClickHouse/issues/20597). [#20610](https://github.com/ClickHouse/ClickHouse/pull/20610) ([flynn](https://github.com/ucasfl)). +* Backported in [#21571](https://github.com/ClickHouse/ClickHouse/issues/21571): `force_drop_table` flag didn't work for `MATERIALIZED VIEW`, it's fixed. Fixes [#18943](https://github.com/ClickHouse/ClickHouse/issues/18943). [#20626](https://github.com/ClickHouse/ClickHouse/pull/20626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix usage of `-Distinct` combinator with `-State` combinator in aggregate functions. [#20866](https://github.com/ClickHouse/ClickHouse/pull/20866) ([Anton Popov](https://github.com/CurtizJ)). +* `USE database;` query did not work when using MySQL 5.7 client to connect to ClickHouse server, it's fixed. Fixes [#18926](https://github.com/ClickHouse/ClickHouse/issues/18926). [#20878](https://github.com/ClickHouse/ClickHouse/pull/20878) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 'Empty task was returned from async task queue' on query cancellation. [#20881](https://github.com/ClickHouse/ClickHouse/pull/20881) ([Azat Khuzhin](https://github.com/azat)). +* Closes [#9969](https://github.com/ClickHouse/ClickHouse/issues/9969). Fixed Brotli http compression error, which reproduced for large data sizes, slightly complicated structure and with json output format. Update Brotli to the latest version to include the "fix rare access to uninitialized data in ring-buffer". [#20991](https://github.com/ClickHouse/ClickHouse/pull/20991) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed behaviour, when `ALTER MODIFY COLUMN` created mutation, that will knowingly fail. [#21007](https://github.com/ClickHouse/ClickHouse/pull/21007) ([Anton Popov](https://github.com/CurtizJ)). +* - Block parallel insertions into storage join. [#21009](https://github.com/ClickHouse/ClickHouse/pull/21009) ([Vladimir C](https://github.com/vdimir)). +* Out of bound memory access was possible when formatting specifically crafted out of range value of type `DateTime64`. This closes [#20494](https://github.com/ClickHouse/ClickHouse/issues/20494). This closes [#20543](https://github.com/ClickHouse/ClickHouse/issues/20543). [#21023](https://github.com/ClickHouse/ClickHouse/pull/21023) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix default_replica_path and default_replica_name values are useless on Replicated(*)MergeTree engine when the engine needs specify other parameters. [#21060](https://github.com/ClickHouse/ClickHouse/pull/21060) ([mxzlxy](https://github.com/mxzlxy)). +* Fix type mismatch issue when using LowCardinality keys in joinGet. This fixes [#21114](https://github.com/ClickHouse/ClickHouse/issues/21114). [#21117](https://github.com/ClickHouse/ClickHouse/pull/21117) ([Amos Bird](https://github.com/amosbird)). +* Fix the metadata leak when the Replicated*MergeTree with custom (non default) ZooKeeper cluster is dropped. [#21119](https://github.com/ClickHouse/ClickHouse/pull/21119) ([fastio](https://github.com/fastio)). +* fix bug related to cast tuple to map. Closes [#21029](https://github.com/ClickHouse/ClickHouse/issues/21029). [#21120](https://github.com/ClickHouse/ClickHouse/pull/21120) ([hexiaoting](https://github.com/hexiaoting)). +* Fix `input_format_null_as_default` take effective when types are nullable. This fixes [#21116](https://github.com/ClickHouse/ClickHouse/issues/21116) . [#21121](https://github.com/ClickHouse/ClickHouse/pull/21121) ([Amos Bird](https://github.com/amosbird)). +* Fixes [#21112](https://github.com/ClickHouse/ClickHouse/issues/21112). Fixed bug that could cause duplicates with insert query (if one of the callbacks came a little too late). [#21138](https://github.com/ClickHouse/ClickHouse/pull/21138) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Now mutations allowed only for table engines that support them (MergeTree family, Memory, MaterializedView). Other engines will report a more clear error. Fixes [#21168](https://github.com/ClickHouse/ClickHouse/issues/21168). [#21183](https://github.com/ClickHouse/ClickHouse/pull/21183) ([alesapin](https://github.com/alesapin)). +* Fix crash in `EXPLAIN` for query with `UNION`. Fixes [#20876](https://github.com/ClickHouse/ClickHouse/issues/20876), [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170). [#21246](https://github.com/ClickHouse/ClickHouse/pull/21246) ([flynn](https://github.com/ucasfl)). +* Fix bug with `join_use_nulls` and joining `TOTALS` from subqueries. This closes [#19362](https://github.com/ClickHouse/ClickHouse/issues/19362) and [#21137](https://github.com/ClickHouse/ClickHouse/issues/21137). [#21248](https://github.com/ClickHouse/ClickHouse/pull/21248) ([Vladimir C](https://github.com/vdimir)). +* Fix redundant reconnects to ZooKeeper and the possibility of two active sessions for a single clickhouse server. Both problems introduced in [#14678](https://github.com/ClickHouse/ClickHouse/issues/14678). [#21264](https://github.com/ClickHouse/ClickHouse/pull/21264) ([alesapin](https://github.com/alesapin)). +* Now `ALTER MODIFY COLUMN` queries will correctly affect changes in partition key, skip indices, TTLs, and so on. Fixes [#13675](https://github.com/ClickHouse/ClickHouse/issues/13675). [#21334](https://github.com/ClickHouse/ClickHouse/pull/21334) ([alesapin](https://github.com/alesapin)). +* Fix error `Bad cast from type ... to DB::ColumnLowCardinality` while inserting into table with `LowCardinality` column from `Values` format. Fixes [#21140](https://github.com/ClickHouse/ClickHouse/issues/21140). [#21357](https://github.com/ClickHouse/ClickHouse/pull/21357) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix SIGSEGV for distributed queries on failures. [#21434](https://github.com/ClickHouse/ClickHouse/pull/21434) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21610](https://github.com/ClickHouse/ClickHouse/issues/21610): Fixed race on SSL object inside SecureSocket in Poco. [#21456](https://github.com/ClickHouse/ClickHouse/pull/21456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix a deadlock in `ALTER DELETE` mutations for non replicated MergeTree table engines when the predicate contains the table itself. Fixes [#20558](https://github.com/ClickHouse/ClickHouse/issues/20558). [#21477](https://github.com/ClickHouse/ClickHouse/pull/21477) ([alesapin](https://github.com/alesapin)). + +#### Build/Testing/Packaging Improvement +* Fixed port clash from test_storage_kerberized_hdfs test. [#19974](https://github.com/ClickHouse/ClickHouse/pull/19974) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix some of the issues found by Coverity. See [#19964](https://github.com/ClickHouse/ClickHouse/issues/19964). [#20010](https://github.com/ClickHouse/ClickHouse/pull/20010) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to build ClickHouse with AVX-2 enabled globally. It gives slight performance benefits on modern CPUs. Not recommended for production and will not be supported as official build for now. [#20180](https://github.com/ClickHouse/ClickHouse/pull/20180) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add some checks in order by fix the bug https://clickhouse-test-reports.s3.yandex.net/20472/5bdc57004682a5e0236ec630546d20ad752c2fde/stress_test_(thread)/stderr.log. [#20516](https://github.com/ClickHouse/ClickHouse/pull/20516) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Print `stdout` and `stderr` to log when failed to start docker in integration tests. Before this PR there was a very short error message in this case which didn't help to investigate the problems:. [#20631](https://github.com/ClickHouse/ClickHouse/pull/20631) ([Vitaly Baranov](https://github.com/vitlibar)). +* Remove most of `sleep` commands from kafka integration tests, waiting for log events instead. Add test for different compression methods. [#21111](https://github.com/ClickHouse/ClickHouse/pull/21111) ([filimonov](https://github.com/filimonov)). +* Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Experimental feature + +* Introduce experimental support for window functions, enabled with `allow_experimental_functions = 1`. This is a preliminary, alpha-quality implementation that is not suitable for production use and will change in backward-incompatible ways in future releases. Please see [the documentation](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/sql-reference/window-functions/index.md#experimental-window-functions) for the list of supported features. [#20337](https://github.com/ClickHouse/ClickHouse/pull/20337) ([Alexander Kuzmenkov](https://github.com/akuzm)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Fix access control manager destruction order"'. [#20394](https://github.com/ClickHouse/ClickHouse/pull/20394) ([alesapin](https://github.com/alesapin)). +* NO CL ENTRY: 'Update argmax.md '. [#20625](https://github.com/ClickHouse/ClickHouse/pull/20625) ([Marvin Taschenberger](https://github.com/Taschenbergerm)). + diff --git a/docs/changelogs/v21.3.20.1-lts.md b/docs/changelogs/v21.3.20.1-lts.md new file mode 100644 index 00000000000..ac8c7d2ece2 --- /dev/null +++ b/docs/changelogs/v21.3.20.1-lts.md @@ -0,0 +1,10 @@ +### ClickHouse release v21.3.20.1-lts FIXME as compared to v21.3.19.1-lts + +#### Bug Fix +* Backported in [#32690](https://github.com/ClickHouse/ClickHouse/issues/32690): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). +* Backported in [#33727](https://github.com/ClickHouse/ClickHouse/issues/33727): Fix null pointer dereference in low cardinality data when deserializing LowCardinality data in the Native format. [#33021](https://github.com/ClickHouse/ClickHouse/pull/33021) ([Harry Lee](https://github.com/HarryLeeIBM)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#32791](https://github.com/ClickHouse/ClickHouse/issues/32791): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). + diff --git a/docs/changelogs/v21.3.3.14-lts.md b/docs/changelogs/v21.3.3.14-lts.md new file mode 100644 index 00000000000..3dbfd7f0a04 --- /dev/null +++ b/docs/changelogs/v21.3.3.14-lts.md @@ -0,0 +1,11 @@ +### ClickHouse release v21.3.3.14-lts FIXME as compared to v21.3.2.5-lts + +#### Bug Fix +* Backported in [#21644](https://github.com/ClickHouse/ClickHouse/issues/21644): Fix that S3 table holds old credentials after config update. [#21457](https://github.com/ClickHouse/ClickHouse/pull/21457) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). +* Backported in [#21854](https://github.com/ClickHouse/ClickHouse/issues/21854): Fix possible error ` Cannot find column` when `optimize_skip_unused_shards` is enabled and zero shards are used. [#21579](https://github.com/ClickHouse/ClickHouse/pull/21579) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21687](https://github.com/ClickHouse/ClickHouse/issues/21687): Fix fsync_part_directory for horizontal merge. [#21642](https://github.com/ClickHouse/ClickHouse/pull/21642) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21796](https://github.com/ClickHouse/ClickHouse/issues/21796): Fix distributed requests cancellation (for example simple select from multiple shards with limit, i.e. `select * from remote('127.{2,3}', system.numbers) limit 100`) with `async_socket_for_remote=1`. [#21643](https://github.com/ClickHouse/ClickHouse/pull/21643) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21811](https://github.com/ClickHouse/ClickHouse/issues/21811): Fix bug for ReplicatedMerge table engines when `ALTER MODIFY COLUMN` query doesn't change the type of decimal column if its size (32 bit or 64 bit) doesn't change. [#21728](https://github.com/ClickHouse/ClickHouse/pull/21728) ([alesapin](https://github.com/alesapin)). +* Backported in [#21883](https://github.com/ClickHouse/ClickHouse/issues/21883): Reverted S3 connection pools. [#21737](https://github.com/ClickHouse/ClickHouse/pull/21737) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#21874](https://github.com/ClickHouse/ClickHouse/issues/21874): Fix incorrect query result (and possible crash) which could happen when `WHERE` or `HAVING` condition is pushed before `GROUP BY`. Fixes [#21773](https://github.com/ClickHouse/ClickHouse/issues/21773). [#21841](https://github.com/ClickHouse/ClickHouse/pull/21841) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.3.4.25-lts.md b/docs/changelogs/v21.3.4.25-lts.md new file mode 100644 index 00000000000..431c498dbed --- /dev/null +++ b/docs/changelogs/v21.3.4.25-lts.md @@ -0,0 +1,18 @@ +### ClickHouse release v21.3.4.25-lts FIXME as compared to v21.3.3.14-lts + +#### Bug Fix +* Backported in [#21928](https://github.com/ClickHouse/ClickHouse/issues/21928): Fix Avro format parsing for Kafka. Fixes [#21437](https://github.com/ClickHouse/ClickHouse/issues/21437). [#21438](https://github.com/ClickHouse/ClickHouse/pull/21438) ([Ilya Golshtein](https://github.com/ilejn)). +* Backported in [#22089](https://github.com/ClickHouse/ClickHouse/issues/22089): In case if query has constant `WHERE` condition, and setting `optimize_skip_unused_shards` enabled, all shards may be skipped and query could return incorrect empty result. [#21550](https://github.com/ClickHouse/ClickHouse/pull/21550) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#21793](https://github.com/ClickHouse/ClickHouse/issues/21793): `std::terminate` was called if there is an error writing data into s3. [#21624](https://github.com/ClickHouse/ClickHouse/pull/21624) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#21746](https://github.com/ClickHouse/ClickHouse/issues/21746): Start accepting connections after DDLWorker and dictionaries initialization. [#21676](https://github.com/ClickHouse/ClickHouse/pull/21676) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21749](https://github.com/ClickHouse/ClickHouse/issues/21749): Fix concurrent `OPTIMIZE` and `DROP` for `ReplicatedMergeTree`. [#21716](https://github.com/ClickHouse/ClickHouse/pull/21716) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21878](https://github.com/ClickHouse/ClickHouse/issues/21878): Fix possible crashes in aggregate functions with combinator Distinct, while using two-level aggregation. This is a follow-up fix of https://github.com/ClickHouse/ClickHouse/pull/18365 . Can only reproduced in production env. No test case available yet. cc @CurtizJ. [#21818](https://github.com/ClickHouse/ClickHouse/pull/21818) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#22051](https://github.com/ClickHouse/ClickHouse/issues/22051): Fix deadlock in first catboost model execution. Closes [#13832](https://github.com/ClickHouse/ClickHouse/issues/13832). [#21844](https://github.com/ClickHouse/ClickHouse/pull/21844) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#21993](https://github.com/ClickHouse/ClickHouse/issues/21993): Fix wrong `ORDER BY` results when a query contains window functions, and optimization for reading in primary key order is applied. Fixes [#21828](https://github.com/ClickHouse/ClickHouse/issues/21828). [#21915](https://github.com/ClickHouse/ClickHouse/pull/21915) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#22180](https://github.com/ClickHouse/ClickHouse/issues/22180): Fix reading the HTTP POST request with "multipart/form-data" content type. [#21936](https://github.com/ClickHouse/ClickHouse/pull/21936) ([Ivan](https://github.com/abyss7)). +* Backported in [#21982](https://github.com/ClickHouse/ClickHouse/issues/21982): Prevent hedged connections overlaps (`Unknown packet 9 from server` error). [#21941](https://github.com/ClickHouse/ClickHouse/pull/21941) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#21980](https://github.com/ClickHouse/ClickHouse/issues/21980): Reverted [#15454](https://github.com/ClickHouse/ClickHouse/issues/15454) that may cause significant increase in memory usage while loading external dictionaries of hashed type. This closes [#21935](https://github.com/ClickHouse/ClickHouse/issues/21935). [#21948](https://github.com/ClickHouse/ClickHouse/pull/21948) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#22139](https://github.com/ClickHouse/ClickHouse/issues/22139): The function `decrypt` was lacking a check for the minimal size of data encrypted in AEAD mode. This closes [#21897](https://github.com/ClickHouse/ClickHouse/issues/21897). [#22064](https://github.com/ClickHouse/ClickHouse/pull/22064) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22185](https://github.com/ClickHouse/ClickHouse/issues/22185): Fix uncaught exception in InterserverIOHTTPHandler. [#22146](https://github.com/ClickHouse/ClickHouse/pull/22146) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22205](https://github.com/ClickHouse/ClickHouse/issues/22205): Use finalize() over next() for nested writers. [#22147](https://github.com/ClickHouse/ClickHouse/pull/22147) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.3.5.42-lts.md b/docs/changelogs/v21.3.5.42-lts.md new file mode 100644 index 00000000000..d8616efd0ff --- /dev/null +++ b/docs/changelogs/v21.3.5.42-lts.md @@ -0,0 +1,27 @@ +### ClickHouse release v21.3.5.42-lts FIXME as compared to v21.3.4.25-lts + +#### Bug Fix +* Backported in [#22335](https://github.com/ClickHouse/ClickHouse/issues/22335): Fix table function `clusterAllReplicas` returns wrong `_shard_num`. close [#21481](https://github.com/ClickHouse/ClickHouse/issues/21481). [#21498](https://github.com/ClickHouse/ClickHouse/pull/21498) ([flynn](https://github.com/ucasfl)). +* Backported in [#22319](https://github.com/ClickHouse/ClickHouse/issues/22319): Remove unknown columns from joined table in where for queries to external database engines (MySQL, PostgreSQL). close [#14614](https://github.com/ClickHouse/ClickHouse/issues/14614), close [#19288](https://github.com/ClickHouse/ClickHouse/issues/19288) (dup), close [#19645](https://github.com/ClickHouse/ClickHouse/issues/19645) (dup). [#21640](https://github.com/ClickHouse/ClickHouse/pull/21640) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#22241](https://github.com/ClickHouse/ClickHouse/issues/22241): Fix scalar subquery index analysis. This fixes [#21717](https://github.com/ClickHouse/ClickHouse/issues/21717) , which was introduced in https://github.com/ClickHouse/ClickHouse/pull/18896 . [#21766](https://github.com/ClickHouse/ClickHouse/pull/21766) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#22150](https://github.com/ClickHouse/ClickHouse/issues/22150): Better error handling and logging in WriteBufferFromS3. [#21836](https://github.com/ClickHouse/ClickHouse/pull/21836) ([Pavel Kovalenko](https://github.com/Jokser)). +* Backported in [#22468](https://github.com/ClickHouse/ClickHouse/issues/22468): In rare case, merge for `CollapsingMergeTree` may create granule with `index_granularity + 1` rows. Because of this, internal check, added in [#18928](https://github.com/ClickHouse/ClickHouse/issues/18928) (affects 21.2 and 21.3), may fail with error `Incomplete granules are not allowed while blocks are granules size`. This error did not allow parts to merge. [#21976](https://github.com/ClickHouse/ClickHouse/pull/21976) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#22203](https://github.com/ClickHouse/ClickHouse/issues/22203): Docker entrypoint: avoid chown of `.` in case when `LOG_PATH` is empty. Closes [#22100](https://github.com/ClickHouse/ClickHouse/issues/22100). [#22102](https://github.com/ClickHouse/ClickHouse/pull/22102) ([filimonov](https://github.com/filimonov)). +* Backported in [#22261](https://github.com/ClickHouse/ClickHouse/issues/22261): Disable `async_socket_for_remote`/`use_hedged_requests` for buggy linux kernels. [#22109](https://github.com/ClickHouse/ClickHouse/pull/22109) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22281](https://github.com/ClickHouse/ClickHouse/issues/22281): Fix waiting for `OPTIMIZE` and `ALTER` queries for `ReplicatedMergeTree` table engines. Now the query will not hang when the table was detached or restarted. [#22118](https://github.com/ClickHouse/ClickHouse/pull/22118) ([alesapin](https://github.com/alesapin)). +* Backported in [#22265](https://github.com/ClickHouse/ClickHouse/issues/22265): Fix the background thread pool name. [#22122](https://github.com/ClickHouse/ClickHouse/pull/22122) ([fastio](https://github.com/fastio)). +* Backported in [#22262](https://github.com/ClickHouse/ClickHouse/issues/22262): Fix error `Invalid number of rows in Chunk` in `JOIN` with `TOTALS` and `arrayJoin`. Closes [#19303](https://github.com/ClickHouse/ClickHouse/issues/19303). [#22129](https://github.com/ClickHouse/ClickHouse/pull/22129) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#22316](https://github.com/ClickHouse/ClickHouse/issues/22316): Fix docker entrypoint in case `http_port` is not in the config. [#22132](https://github.com/ClickHouse/ClickHouse/pull/22132) ([Ewout](https://github.com/devwout)). +* Backported in [#22313](https://github.com/ClickHouse/ClickHouse/issues/22313): Fix query cancellation with `use_hedged_requests=0` and `async_socket_for_remote=1`. [#22183](https://github.com/ClickHouse/ClickHouse/pull/22183) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22366](https://github.com/ClickHouse/ClickHouse/issues/22366): Now clickhouse will not throw `LOGICAL_ERROR` exception when we try to mutate the already covered part. Fixes [#22013](https://github.com/ClickHouse/ClickHouse/issues/22013). [#22291](https://github.com/ClickHouse/ClickHouse/pull/22291) ([alesapin](https://github.com/alesapin)). +* Backported in [#22562](https://github.com/ClickHouse/ClickHouse/issues/22562): Do not limit HTTP chunk size. Fixes [#21907](https://github.com/ClickHouse/ClickHouse/issues/21907). [#22322](https://github.com/ClickHouse/ClickHouse/pull/22322) ([Ivan](https://github.com/abyss7)). +* Backported in [#22531](https://github.com/ClickHouse/ClickHouse/issues/22531): Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22463](https://github.com/ClickHouse/ClickHouse/issues/22463): Add (missing) memory accounting in parallel parsing routines. In previous versions OOM was possible when the resultset contains very large blocks of data. This closes [#22008](https://github.com/ClickHouse/ClickHouse/issues/22008). [#22425](https://github.com/ClickHouse/ClickHouse/pull/22425) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22507](https://github.com/ClickHouse/ClickHouse/issues/22507): Remove socket from epoll before cancelling packet receiver in HedgedConnections to prevent possible race. I hope it fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#22554](https://github.com/ClickHouse/ClickHouse/issues/22554): Fix bug in partial merge join with `LowCardinality`. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#22607](https://github.com/ClickHouse/ClickHouse/issues/22607): Fix deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22576](https://github.com/ClickHouse/ClickHouse/issues/22576): Fix UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22651](https://github.com/ClickHouse/ClickHouse/issues/22651): Avoid UB in *Log engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22653](https://github.com/ClickHouse/ClickHouse/issues/22653): Try flush write buffer only if it is initialized. Fixes segfault when client closes connection very early [#22579](https://github.com/ClickHouse/ClickHouse/issues/22579). [#22591](https://github.com/ClickHouse/ClickHouse/pull/22591) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#22681](https://github.com/ClickHouse/ClickHouse/issues/22681): Fix LOGICAL_ERROR for Log with nested types w/o columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.3.6.55-lts.md b/docs/changelogs/v21.3.6.55-lts.md new file mode 100644 index 00000000000..68980af6264 --- /dev/null +++ b/docs/changelogs/v21.3.6.55-lts.md @@ -0,0 +1,19 @@ +### ClickHouse release v21.3.6.55-lts FIXME as compared to v21.3.5.42-lts + +#### Improvement +* Backported in [#22811](https://github.com/ClickHouse/ClickHouse/issues/22811): If PODArray was instantiated with element size that is neither a fraction or a multiple of 16, buffer overflow was possible. No bugs in current releases exist. [#21533](https://github.com/ClickHouse/ClickHouse/pull/21533) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22823](https://github.com/ClickHouse/ClickHouse/issues/22823): Add profile event HedgedRequestsChangeReplica, change read data timeout from sec to ms. [#21886](https://github.com/ClickHouse/ClickHouse/pull/21886) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#22936](https://github.com/ClickHouse/ClickHouse/issues/22936): Correctly check structure of async distributed blocks. [#22325](https://github.com/ClickHouse/ClickHouse/pull/22325) ([Azat Khuzhin](https://github.com/azat)). + +#### Bug Fix +* Backported in [#22968](https://github.com/ClickHouse/ClickHouse/issues/22968): Fix very rare bug when quorum insert with `quorum_parallel=1` is not really "quorum" because of deduplication. [#18215](https://github.com/ClickHouse/ClickHouse/pull/18215) ([filimonov](https://github.com/filimonov)). +* Backported in [#22723](https://github.com/ClickHouse/ClickHouse/issues/22723): Check if table function view is used as a column. This complements https://github.com/ClickHouse/ClickHouse/pull/20350. [#21465](https://github.com/ClickHouse/ClickHouse/pull/21465) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#22807](https://github.com/ClickHouse/ClickHouse/issues/22807): Follow-up fix for [#21936](https://github.com/ClickHouse/ClickHouse/issues/21936). Also fixes [#22433](https://github.com/ClickHouse/ClickHouse/issues/22433). [#22518](https://github.com/ClickHouse/ClickHouse/pull/22518) ([Ivan](https://github.com/abyss7)). +* Backported in [#22759](https://github.com/ClickHouse/ClickHouse/issues/22759): Fix usage of function `map` in distributed queries. [#22588](https://github.com/ClickHouse/ClickHouse/pull/22588) ([foolchi](https://github.com/foolchi)). +* Backported in [#22701](https://github.com/ClickHouse/ClickHouse/issues/22701): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). +* Backported in [#22738](https://github.com/ClickHouse/ClickHouse/issues/22738): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#22892](https://github.com/ClickHouse/ClickHouse/issues/22892): Fix approx total rows accounting for reverse reading from MergeTree. [#22726](https://github.com/ClickHouse/ClickHouse/pull/22726) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22922](https://github.com/ClickHouse/ClickHouse/issues/22922): Fix pushdown of `HAVING` in case, when filter column is used in aggregation. [#22763](https://github.com/ClickHouse/ClickHouse/pull/22763) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#22916](https://github.com/ClickHouse/ClickHouse/issues/22916): LIVE VIEW (experimental feature). Fix possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, see https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e. [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#22920](https://github.com/ClickHouse/ClickHouse/issues/22920): Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.3.7.62-stable.md b/docs/changelogs/v21.3.7.62-stable.md new file mode 100644 index 00000000000..df919be0f42 --- /dev/null +++ b/docs/changelogs/v21.3.7.62-stable.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.3.7.62-stable FIXME as compared to v21.3.6.55-lts + +#### Improvement +* Backported in [#23014](https://github.com/ClickHouse/ClickHouse/issues/23014): Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23081](https://github.com/ClickHouse/ClickHouse/issues/23081): Raised the threshold on max number of matches in result of the function `extractAllGroupsHorizontal`. [#23036](https://github.com/ClickHouse/ClickHouse/pull/23036) ([Vasily Nemkov](https://github.com/Enmk)). + +#### Bug Fix +* Backported in [#23156](https://github.com/ClickHouse/ClickHouse/issues/23156): Fixed a bug with unlimited wait for auxiliary AWS requests. [#22594](https://github.com/ClickHouse/ClickHouse/pull/22594) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#23033](https://github.com/ClickHouse/ClickHouse/issues/23033): Fix error `Cannot find column in ActionsDAG result` which may happen if subquery uses `untuple`. Fixes [#22290](https://github.com/ClickHouse/ClickHouse/issues/22290). [#22991](https://github.com/ClickHouse/ClickHouse/pull/22991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#23075](https://github.com/ClickHouse/ClickHouse/issues/23075): Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23170](https://github.com/ClickHouse/ClickHouse/issues/23170): Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.3.8.76-lts.md b/docs/changelogs/v21.3.8.76-lts.md new file mode 100644 index 00000000000..2002d9e3e1f --- /dev/null +++ b/docs/changelogs/v21.3.8.76-lts.md @@ -0,0 +1,21 @@ +### ClickHouse release v21.3.8.76-lts FIXME as compared to v21.3.7.62-stable + +#### Improvement +* Backported in [#23396](https://github.com/ClickHouse/ClickHouse/issues/23396): If tuple of NULLs, e.g. `(NULL, NULL)` is on the left hand side of `IN` operator with tuples of non-NULLs on the right hand side, e.g. `SELECT (NULL, NULL) IN ((0, 0), (3, 1))` return 0 instead of throwing an exception about incompatible types. The expression may also appear due to optimization of something like `SELECT (NULL, NULL) = (8, 0) OR (NULL, NULL) = (3, 2) OR (NULL, NULL) = (0, 0) OR (NULL, NULL) = (3, 1)`. This closes [#22017](https://github.com/ClickHouse/ClickHouse/issues/22017). [#22063](https://github.com/ClickHouse/ClickHouse/pull/22063) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23272](https://github.com/ClickHouse/ClickHouse/issues/23272): Disable settings `use_hedged_requests` and `async_socket_for_remote` because there is an evidence that it may cause issues. [#23261](https://github.com/ClickHouse/ClickHouse/pull/23261) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#23232](https://github.com/ClickHouse/ClickHouse/issues/23232): Server might fail to start if `data_type_default_nullable` setting is enabled in default profile, it's fixed. Fixes [#22573](https://github.com/ClickHouse/ClickHouse/issues/22573). [#23185](https://github.com/ClickHouse/ClickHouse/pull/23185) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23406](https://github.com/ClickHouse/ClickHouse/issues/23406): QueryAliasVisitor to prefer alias for ASTWithAlias if subquery was optimized to constant. Fixes [#22924](https://github.com/ClickHouse/ClickHouse/issues/22924). Fixes [#10401](https://github.com/ClickHouse/ClickHouse/issues/10401). [#23191](https://github.com/ClickHouse/ClickHouse/pull/23191) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23381](https://github.com/ClickHouse/ClickHouse/issues/23381): Fixed `Not found column` error when selecting from `MaterializeMySQL` with condition on key column. Fixes [#22432](https://github.com/ClickHouse/ClickHouse/issues/22432). [#23200](https://github.com/ClickHouse/ClickHouse/pull/23200) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23284](https://github.com/ClickHouse/ClickHouse/issues/23284): Fixed simple key dictionary from DDL creation if primary key is not first attribute. Fixes [#23236](https://github.com/ClickHouse/ClickHouse/issues/23236). [#23262](https://github.com/ClickHouse/ClickHouse/pull/23262) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23450](https://github.com/ClickHouse/ClickHouse/issues/23450): Fixed very rare (distributed) race condition between creation and removal of ReplicatedMergeTree tables. It might cause exceptions like `node doesn't exist` on attempt to create replicated table. Fixes [#21419](https://github.com/ClickHouse/ClickHouse/issues/21419). [#23294](https://github.com/ClickHouse/ClickHouse/pull/23294) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23500](https://github.com/ClickHouse/ClickHouse/issues/23500): Fix possible crash in case if `unknown packet` was received form remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#23468](https://github.com/ClickHouse/ClickHouse/issues/23468): `ORDER BY` with `COLLATE` was not working correctly if the column is in primary key (or is a monotonic function of it) and the setting `optimize_read_in_order` is not turned off. This closes [#22379](https://github.com/ClickHouse/ClickHouse/issues/22379). Workaround for older versions: turn the setting `optimize_read_in_order` off. [#23375](https://github.com/ClickHouse/ClickHouse/pull/23375) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23544](https://github.com/ClickHouse/ClickHouse/issues/23544): Remove support for `argMin` and `argMax` for single `Tuple` argument. The code was not memory-safe. The feature was added by mistake and it is confusing for people. These functions can be reintroduced under different names later. This fixes [#22384](https://github.com/ClickHouse/ClickHouse/issues/22384) and reverts [#17359](https://github.com/ClickHouse/ClickHouse/issues/17359). [#23393](https://github.com/ClickHouse/ClickHouse/pull/23393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23479](https://github.com/ClickHouse/ClickHouse/issues/23479): Kafka storage may support `arrow` and `arrowstream` format messages. [#23415](https://github.com/ClickHouse/ClickHouse/pull/23415) ([Chao Ma](https://github.com/godliness)). +* Backported in [#23499](https://github.com/ClickHouse/ClickHouse/issues/23499): - Bug fix for `deltaSum` aggregate function in counter reset case ... [#23437](https://github.com/ClickHouse/ClickHouse/pull/23437) ([Russ Frank](https://github.com/rf)). +* Backported in [#23493](https://github.com/ClickHouse/ClickHouse/issues/23493): Fix bug that does not allow cast from empty array literal, to array with dimensions greater than 1. Closes [#14476](https://github.com/ClickHouse/ClickHouse/issues/14476). [#23456](https://github.com/ClickHouse/ClickHouse/pull/23456) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23536](https://github.com/ClickHouse/ClickHouse/issues/23536): When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#23533](https://github.com/ClickHouse/ClickHouse/issues/23533): Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.9.83-lts.md b/docs/changelogs/v21.3.9.83-lts.md new file mode 100644 index 00000000000..e437ee4800f --- /dev/null +++ b/docs/changelogs/v21.3.9.83-lts.md @@ -0,0 +1,15 @@ +### ClickHouse release v21.3.9.83-lts FIXME as compared to v21.3.8.76-lts + +#### Improvement +* Backported in [#23680](https://github.com/ClickHouse/ClickHouse/issues/23680): Fixed `quantile(s)TDigest`. Added special handling of singleton centroids according to tdunning/t-digest 3.2+. Also a bug with over-compression of centroids in implementation of earlier version of the algorithm was fixed. [#23314](https://github.com/ClickHouse/ClickHouse/pull/23314) ([Vladimir Chebotarev](https://github.com/excitoon)). + +#### Bug Fix +* Backported in [#23579](https://github.com/ClickHouse/ClickHouse/issues/23579): Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23677](https://github.com/ClickHouse/ClickHouse/issues/23677): Don't relax NOT conditions during partition pruning. This fixes [#23305](https://github.com/ClickHouse/ClickHouse/issues/23305) and [#21539](https://github.com/ClickHouse/ClickHouse/issues/21539). [#23310](https://github.com/ClickHouse/ClickHouse/pull/23310) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#23582](https://github.com/ClickHouse/ClickHouse/issues/23582): * Fix bug in dict join with join_algorithm = 'auto'. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#23585](https://github.com/ClickHouse/ClickHouse/issues/23585): Allow to move more conditions to `PREWHERE` as it was before version 21.1. Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#23592](https://github.com/ClickHouse/ClickHouse/issues/23592): Fixed `Cannot unlink file` error on unsuccessful creation of ReplicatedMergeTree table with multidisk configuration. This closes [#21755](https://github.com/ClickHouse/ClickHouse/issues/21755). [#23433](https://github.com/ClickHouse/ClickHouse/pull/23433) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23590](https://github.com/ClickHouse/ClickHouse/issues/23590): Fix corner cases in vertical merges with `ReplacingMergeTree`. In rare cases they could lead to fails of merges with exceptions like `Incomplete granules are not allowed while blocks are granules size`. [#23459](https://github.com/ClickHouse/ClickHouse/pull/23459) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#23612](https://github.com/ClickHouse/ClickHouse/issues/23612): Fix restart / stop command hanging. Closes [#20214](https://github.com/ClickHouse/ClickHouse/issues/20214). [#23552](https://github.com/ClickHouse/ClickHouse/pull/23552) ([filimonov](https://github.com/filimonov)). +* Backported in [#23696](https://github.com/ClickHouse/ClickHouse/issues/23696): Added an exception in case of completely the same values in both samples in aggregate function `mannWhitneyUTest`. This fixes [#23646](https://github.com/ClickHouse/ClickHouse/issues/23646). [#23654](https://github.com/ClickHouse/ClickHouse/pull/23654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.4.1.6422-prestable.md b/docs/changelogs/v21.4.1.6422-prestable.md new file mode 100644 index 00000000000..8cd10834fae --- /dev/null +++ b/docs/changelogs/v21.4.1.6422-prestable.md @@ -0,0 +1,173 @@ +### ClickHouse release v21.4.1.6422-prestable FIXME as compared to v21.3.1.6185-prestable + +#### Backward Incompatible Change +* Now replicas that are processing the `ALTER TABLE ATTACH PART[ITION]` command search in their `detached/` folders before fetching the data from other replicas. As an implementation detail, a new command `ATTACH_PART` is introduced in the replicated log. Parts are searched and compared by their checksums. [#18978](https://github.com/ClickHouse/ClickHouse/pull/18978) ([Mike Kot](https://github.com/myrrc)). +* Column `keys` in table `system.dictionaries` was replaced to columns `key.names` and `key.types`. Columns `key.names`, `key.types`, `attribute.names`, `attribute.types` from `system.dictionaries` table does not require dictionary to be loaded. [#21884](https://github.com/ClickHouse/ClickHouse/pull/21884) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix `cutToFirstSignificantSubdomainCustom()`/`firstSignificantSubdomainCustom()` returning wrong result for 3+ level domains present in custom top-level domain list. For input domains matching these custom top-level domains, the third-level domain was considered to be the first significant one. This is now fixed. This change may introduce incompatibility if the function is used in e.g. the sharding key. [#21946](https://github.com/ClickHouse/ClickHouse/pull/21946) ([Azat Khuzhin](https://github.com/azat)). +* The `toStartOfIntervalFunction` will align hour intervals to the midnight (in previous versions they were aligned to the start of unix epoch). For example, `toStartOfInterval(x, INTERVAL 11 HOUR)` will split every day into three intervals: 00:00:00..10:59:59, 11:00:00..21:59:59 and 22:00:00..23:59:59. This behaviour is more suited for practical needs. This closes [#9510](https://github.com/ClickHouse/ClickHouse/issues/9510). [#22060](https://github.com/ClickHouse/ClickHouse/pull/22060) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* Extended range of `DateTime64` to properly support dates from year 1925 to 2283. Improved support of `DateTime` around zero date (`1970-01-01`). ... [#9404](https://github.com/ClickHouse/ClickHouse/pull/9404) ([Vasily Nemkov](https://github.com/Enmk)). +* - Added support of Kerberos authentication for preconfigured users and HTTP requests (GSS-SPNEGO). [#14995](https://github.com/ClickHouse/ClickHouse/pull/14995) ([Denis Glazachev](https://github.com/traceon)). +* Zero-copy replication for ReplicatedMergeTree over S3 storage. [#16240](https://github.com/ClickHouse/ClickHouse/pull/16240) ([ianton-ru](https://github.com/ianton-ru)). +* Support `dictHas` function for `RangeHashedDictionary`. Fixes [#6680](https://github.com/ClickHouse/ClickHouse/issues/6680). [#19816](https://github.com/ClickHouse/ClickHouse/pull/19816) ([Maksim Kita](https://github.com/kitaisreal)). +* Supports implicit key type conversion for JOIN. Closes [#18567](https://github.com/ClickHouse/ClickHouse/issues/18567). [#19885](https://github.com/ClickHouse/ClickHouse/pull/19885) ([Vladimir C](https://github.com/vdimir)). +* Allow customizing timeouts for http connections used for replication independently from other http timeouts. [#20088](https://github.com/ClickHouse/ClickHouse/pull/20088) ([nvartolomei](https://github.com/nvartolomei)). +* Added async update in `ComplexKeyCache`, `SSDCache`, `SSDComplexKeyCache` dictionaries. Added support for Nullable type in `Cache`, `ComplexKeyCache`, `SSDCache`, `SSDComplexKeyCache` dictionaries. Added support for multiple attributes fetch with `dictGet`, `dictGetOrDefault` functions. Fixes [#21517](https://github.com/ClickHouse/ClickHouse/issues/21517). [#20595](https://github.com/ClickHouse/ClickHouse/pull/20595) ([Maksim Kita](https://github.com/kitaisreal)). +* Added `Grant,` `Revoke` and `System` values of `query_kind` column for corresponding queries in `system.query_log` ... [#21102](https://github.com/ClickHouse/ClickHouse/pull/21102) ([Vasily Nemkov](https://github.com/Enmk)). +* Added new SQL command ALTER TABLE 'table_name' UNFREEZE [PARTITION 'part_expr'] WITH NAME 'backup_name'. [#21142](https://github.com/ClickHouse/ClickHouse/pull/21142) ([Pavel Kovalenko](https://github.com/Jokser)). +* Added ExecutablePool dictionary source. Close [#14528](https://github.com/ClickHouse/ClickHouse/issues/14528). [#21321](https://github.com/ClickHouse/ClickHouse/pull/21321) ([Maksim Kita](https://github.com/kitaisreal)). +* - Add function `isIPAddressInRange` to test if an IPv4 or IPv6 address is contained in a given CIDR network prefix. [#21329](https://github.com/ClickHouse/ClickHouse/pull/21329) ([PHO](https://github.com/depressed-pho)). +* Add `_partition_id` virtual column for `MergeTree*` engines. Allow to prune partitions by `_partition_id`. Add `partitionID()` function to calculate partition id string. [#21401](https://github.com/ClickHouse/ClickHouse/pull/21401) ([Amos Bird](https://github.com/amosbird)). +* Add new column `slowdowns_count` to `system.clusters`. When using hedged requests, it shows how many times we switched to another replica because this replica was responding slowly. Also show actual value of `errors_count` in `system.clusters`. [#21480](https://github.com/ClickHouse/ClickHouse/pull/21480) ([Kruglov Pavel](https://github.com/Avogar)). +* Add option `--backslash` for clickhouse-format, which can add a backslash at the end of each line of the formatted query. [#21494](https://github.com/ClickHouse/ClickHouse/pull/21494) ([flynn](https://github.com/ucasfl)). +* Add new optional clause GRANTEES for CREATE/ALTER USER commands:. [#21641](https://github.com/ClickHouse/ClickHouse/pull/21641) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add `ctime` option to `zookeeper-dump-tree`. It allows to dump node creation time. [#21842](https://github.com/ClickHouse/ClickHouse/pull/21842) ([Ilya](https://github.com/HumanUser)). +* Functions 'dictGet', 'dictHas' use current database name if it is not specified for dictionaries created with DDL. Closes [#21632](https://github.com/ClickHouse/ClickHouse/issues/21632). [#21859](https://github.com/ClickHouse/ClickHouse/pull/21859) ([Maksim Kita](https://github.com/kitaisreal)). +* Support `Nullable` type for `PolygonDictionary` attribute. [#21890](https://github.com/ClickHouse/ClickHouse/pull/21890) ([Maksim Kita](https://github.com/kitaisreal)). +* Added table function `dictionary`. It works the same way as `Dictionary` engine. Closes [#21560](https://github.com/ClickHouse/ClickHouse/issues/21560). [#21910](https://github.com/ClickHouse/ClickHouse/pull/21910) ([Maksim Kita](https://github.com/kitaisreal)). +* Add function `timezoneOf` that returns the timezone name of `DateTime` or `DateTime64` data types. This does not close [#9959](https://github.com/ClickHouse/ClickHouse/issues/9959). Fix inconsistencies in function names: add aliases `timezone` and `timeZone` as well as `toTimezone` and `toTimeZone` and `timezoneOf` and `timeZoneOf`. [#22001](https://github.com/ClickHouse/ClickHouse/pull/22001) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `prefer_column_name_to_alias` setting to use original column names instead of aliases. it is needed to be more compatible with common databases' aliasing rules. This is for [#9715](https://github.com/ClickHouse/ClickHouse/issues/9715) and [#9887](https://github.com/ClickHouse/ClickHouse/issues/9887). [#22044](https://github.com/ClickHouse/ClickHouse/pull/22044) ([Amos Bird](https://github.com/amosbird)). +* Improved performance of `dictGetHierarchy`, `dictIsIn` functions. Added functions `dictGetChildren(dictionary, key)`, `dictGetDescendants(dictionary, key, level)`. Function `dictGetChildren` return all children as an array if indexes. It is a inverse transformation for `dictGetHierarchy`. Function `dictGetDescendants` return all descendants as if `dictGetChildren` was applied `level` times recursively. Zero `level` value is equivalent to infinity. Closes [#14656](https://github.com/ClickHouse/ClickHouse/issues/14656). [#22096](https://github.com/ClickHouse/ClickHouse/pull/22096) ([Maksim Kita](https://github.com/kitaisreal)). +* Added function `dictGetOrNull`. It works like `dictGet`, but return `Null` in case key was not found in dictionary. Closes [#22375](https://github.com/ClickHouse/ClickHouse/issues/22375). [#22413](https://github.com/ClickHouse/ClickHouse/pull/22413) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Performance Improvement +* Support parallel parsing for `CSVWithNames` and `TSVWithNames` formats. This closes [#21085](https://github.com/ClickHouse/ClickHouse/issues/21085). [#21149](https://github.com/ClickHouse/ClickHouse/pull/21149) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Improved performance by replacing `memcpy` to another implementation. This closes [#18583](https://github.com/ClickHouse/ClickHouse/issues/18583). [#21520](https://github.com/ClickHouse/ClickHouse/pull/21520) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Supported parallel formatting in clickhouse-local and everywhere else. [#21630](https://github.com/ClickHouse/ClickHouse/pull/21630) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Optimize performance of queries like `SELECT ... FINAL ... WHERE`. Now in queries with `FINAL` it's allowed to move to `PREWHERE` columns, which are in sorting key. ... [#21830](https://github.com/ClickHouse/ClickHouse/pull/21830) ([foolchi](https://github.com/foolchi)). +* Faster `GROUP BY` with small `max_rows_to_group_by` and `group_by_overflow_mode='any'`. [#21856](https://github.com/ClickHouse/ClickHouse/pull/21856) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Avoid unnecessary data copy when using codec `NONE`. Please note that codec `NONE` is mostly useless - it's recommended to always use compression (`LZ4` is by default). Despite the common belief, disabling compression may not improve performance (the opposite effect is possible). The `NONE` codec is useful in some cases: - when data is uncompressable; - for synthetic benchmarks. [#22145](https://github.com/ClickHouse/ClickHouse/pull/22145) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add cache for files read with `min_bytes_to_use_mmap_io` setting. It makes significant (2x and more) performance improvement when the value of the setting is small by avoiding frequent mmap/munmap calls and the consequent page faults. Note that mmap IO has major drawbacks that makes it less reliable in production (e.g. hung or SIGBUS on faulty disks; less controllable memory usage). Nevertheless it is good in benchmarks. [#22206](https://github.com/ClickHouse/ClickHouse/pull/22206) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable read with mmap IO for file ranges from 64 MiB (the settings `min_bytes_to_use_mmap_io`). It may lead to moderate performance improvement. [#22326](https://github.com/ClickHouse/ClickHouse/pull/22326) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Introduce a new merge tree setting `min_bytes_to_rebalance_partition_over_jbod` which allows assigning new parts to different disks of a JBOD volume in a balanced way. [#16481](https://github.com/ClickHouse/ClickHouse/pull/16481) ([Amos Bird](https://github.com/amosbird)). +* Improve performance of aggregation in order of sorting key (with enabled setting `optimize_aggregation_in_order`). [#19401](https://github.com/ClickHouse/ClickHouse/pull/19401) ([Anton Popov](https://github.com/CurtizJ)). +* MaterializeMySQL: add minmax skipping index for _version column. [#20382](https://github.com/ClickHouse/ClickHouse/pull/20382) ([Stig Bakken](https://github.com/stigsb)). +* Do not create empty parts on INSERT when `optimize_on_insert` setting enabled. Fixes [#20304](https://github.com/ClickHouse/ClickHouse/issues/20304). [#20387](https://github.com/ClickHouse/ClickHouse/pull/20387) ([Kruglov Pavel](https://github.com/Avogar)). +* - Support more cases to rewrite `CROSS JOIN` to `INNER JOIN`. [#20392](https://github.com/ClickHouse/ClickHouse/pull/20392) ([Vladimir C](https://github.com/vdimir)). +* MaterializeMySQL: Attempt to reconnect to MySQL if the connection is lost. [#20961](https://github.com/ClickHouse/ClickHouse/pull/20961) ([Håvard Kvålen](https://github.com/havardk)). +* Improve support of integer keys in data type `Map`. [#21157](https://github.com/ClickHouse/ClickHouse/pull/21157) ([Anton Popov](https://github.com/CurtizJ)). +* Improve clickhouse-format to not throw exception when there are extra spaces or comment after the last query, and throw exception early with readable message when format `ASTInsertQuery` with data . [#21311](https://github.com/ClickHouse/ClickHouse/pull/21311) ([flynn](https://github.com/ucasfl)). +* Age and Precision in graphite rollup configs should increase from retention to retention. Now it's checked and the wrong config raises an exception. [#21496](https://github.com/ClickHouse/ClickHouse/pull/21496) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add setting `optimize_skip_unused_shards_limit` to limit the number of sharding key values for `optimize_skip_unused_shards`. [#21512](https://github.com/ClickHouse/ClickHouse/pull/21512) ([Azat Khuzhin](https://github.com/azat)). +* Add aliases `simpleJSONExtract/simpleJSONHas` to `visitParam/visitParamExtract{UInt, Int, Bool, Float, Raw, String}`. Fixes [#21383](https://github.com/ClickHouse/ClickHouse/issues/21383). [#21519](https://github.com/ClickHouse/ClickHouse/pull/21519) ([fastio](https://github.com/fastio)). +* Add `last_error_time`/`last_error_message`/`last_error_stacktrace`/`remote` columns for `system.errors`. [#21529](https://github.com/ClickHouse/ClickHouse/pull/21529) ([Azat Khuzhin](https://github.com/azat)). +* If PODArray was instantiated with element size that is neither a fraction or a multiple of 16, buffer overflow was possible. No bugs in current releases exist. [#21533](https://github.com/ClickHouse/ClickHouse/pull/21533) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* - Propagate query and session settings for distributed DDL queries. Set `distributed_ddl_entry_format_version` to 2 to enable this. - Added `distributed_ddl_output_mode` setting. Supported modes: `none`, `throw` (default), `null_status_on_timeout` and `never_throw`. - Miscellaneous fixes and improvements for `Replicated` database engine. [#21535](https://github.com/ClickHouse/ClickHouse/pull/21535) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update clusters only if their configurations were updated. [#21685](https://github.com/ClickHouse/ClickHouse/pull/21685) ([Kruglov Pavel](https://github.com/Avogar)). +* Support replicas priority for postgres dictionary source. [#21710](https://github.com/ClickHouse/ClickHouse/pull/21710) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Closes [#21701](https://github.com/ClickHouse/ClickHouse/issues/21701). Support non-default table schema for postgres storage/table-function. [#21711](https://github.com/ClickHouse/ClickHouse/pull/21711) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better formatting for `Array` and `Map` data types in Web UI. [#21798](https://github.com/ClickHouse/ClickHouse/pull/21798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* DiskS3 (experimental feature under development). Fixed bug with the impossibility to move directory if the destination is not empty and cache disk is used. [#21837](https://github.com/ClickHouse/ClickHouse/pull/21837) ([Pavel Kovalenko](https://github.com/Jokser)). +* Add connection pool for PostgreSQL table/database engine and dictionary source. Should fix [#21444](https://github.com/ClickHouse/ClickHouse/issues/21444). [#21839](https://github.com/ClickHouse/ClickHouse/pull/21839) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add profile event HedgedRequestsChangeReplica, change read data timeout from sec to ms. [#21886](https://github.com/ClickHouse/ClickHouse/pull/21886) ([Kruglov Pavel](https://github.com/Avogar)). +* Support `RANGE OFFSET` frame for floating point types. Implement `lagInFrame`/`leadInFrame` window functions, which are analogous to `lag`/`lead`, but respect the window frame. They are identical when the frame is `between unbounded preceding and unbounded following`. This closes [#5485](https://github.com/ClickHouse/ClickHouse/issues/5485). [#21895](https://github.com/ClickHouse/ClickHouse/pull/21895) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Show path to data directory of `EmbeddedRocksDB` tables in system tables. [#21903](https://github.com/ClickHouse/ClickHouse/pull/21903) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Supported `replication_alter_partitions_sync=1` setting for moving partitions from helping table to destination. Decreased default timeouts. Fixes [#21911](https://github.com/ClickHouse/ClickHouse/issues/21911). [#21912](https://github.com/ClickHouse/ClickHouse/pull/21912) ([jasong](https://github.com/songenjie)). +* If partition key of a `MergeTree` table does not include `Date` or `DateTime` columns but includes exactly one `DateTime64` column, expose its values in the `min_time` and `max_time` columns in `system.parts` and `system.parts_columns` tables. Add `min_time` and `max_time` columns to `system.parts_columns` table (these was inconsistency to the `system.parts` table). This closes [#18244](https://github.com/ClickHouse/ClickHouse/issues/18244). [#22011](https://github.com/ClickHouse/ClickHouse/pull/22011) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* * Add option strict_increase to windowFunnel function to calculate each event once (resolve [#21835](https://github.com/ClickHouse/ClickHouse/issues/21835)). [#22025](https://github.com/ClickHouse/ClickHouse/pull/22025) ([Vladimir C](https://github.com/vdimir)). +* Added case insensitive aliases for `CONNECTION_ID()` and `VERSION()` functions. This fixes [#22028](https://github.com/ClickHouse/ClickHouse/issues/22028). [#22042](https://github.com/ClickHouse/ClickHouse/pull/22042) ([Eugene Klimov](https://github.com/Slach)). +* Update used version of simdjson to 0.9.1. This fixes [#21984](https://github.com/ClickHouse/ClickHouse/issues/21984). [#22057](https://github.com/ClickHouse/ClickHouse/pull/22057) ([Vitaly Baranov](https://github.com/vitlibar)). +* Convert `system.errors.stack_trace` from `String` into `Array(UInt64)` (This should decrease overhead for the errors collecting). [#22058](https://github.com/ClickHouse/ClickHouse/pull/22058) ([Azat Khuzhin](https://github.com/azat)). +* If tuple of NULLs, e.g. `(NULL, NULL)` is on the left hand side of `IN` operator with tuples of non-NULLs on the right hand side, e.g. `SELECT (NULL, NULL) IN ((0, 0), (3, 1))` return 0 instead of throwing an exception about incompatible types. The expression may also appear due to optimization of something like `SELECT (NULL, NULL) = (8, 0) OR (NULL, NULL) = (3, 2) OR (NULL, NULL) = (0, 0) OR (NULL, NULL) = (3, 1)`. This closes [#22017](https://github.com/ClickHouse/ClickHouse/issues/22017). [#22063](https://github.com/ClickHouse/ClickHouse/pull/22063) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added possibility to migrate existing S3 disk to the schema with backup-restore capabilities. [#22070](https://github.com/ClickHouse/ClickHouse/pull/22070) ([Pavel Kovalenko](https://github.com/Jokser)). +* Add case-insensitive history search/navigation and subword movement features to clickhouse-client. [#22105](https://github.com/ClickHouse/ClickHouse/pull/22105) ([Amos Bird](https://github.com/amosbird)). +* Add `current_database` column to `system.processes` table. It contains the current database of the query. [#22365](https://github.com/ClickHouse/ClickHouse/pull/22365) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix MSan report for function `range` with `UInt256` argument (support for large integers is experimental). This closes [#22157](https://github.com/ClickHouse/ClickHouse/issues/22157). [#22387](https://github.com/ClickHouse/ClickHouse/pull/22387) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error `Directory tmp_fetch_XXX already exists` which could happen after failed fetch part. Delete temporary fetch directory if it already exists. Fixes [#14197](https://github.com/ClickHouse/ClickHouse/issues/14197). [#22411](https://github.com/ClickHouse/ClickHouse/pull/22411) ([nvartolomei](https://github.com/nvartolomei)). +* Better exception message in client in case of exception while server is writing blocks. In previous versions client may get misleading message like `Data compressed with different methods`. [#22427](https://github.com/ClickHouse/ClickHouse/pull/22427) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Fixed open behavior of remote host filter in case when there is `remote_url_allow_hosts` section in configuration but no entries there. :warning: please add a note about potential issue when upgrading - @alexey-milovidov. [#20058](https://github.com/ClickHouse/ClickHouse/pull/20058) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix name clashes in `PredicateRewriteVisitor`. It caused incorrect `WHERE` filtration after full join. Close [#20497](https://github.com/ClickHouse/ClickHouse/issues/20497). [#20622](https://github.com/ClickHouse/ClickHouse/pull/20622) ([Vladimir C](https://github.com/vdimir)). +* `force_drop_table` flag didn't work for `MATERIALIZED VIEW`, it's fixed. Fixes [#18943](https://github.com/ClickHouse/ClickHouse/issues/18943). [#20626](https://github.com/ClickHouse/ClickHouse/pull/20626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix official website documents which introduced cluster secret feature. [#21331](https://github.com/ClickHouse/ClickHouse/pull/21331) ([Chao Ma](https://github.com/godliness)). +* Fix receive and send timeouts and non-blocking read in secure socket. [#21429](https://github.com/ClickHouse/ClickHouse/pull/21429) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix Avro format parsing for Kafka. Fixes [#21437](https://github.com/ClickHouse/ClickHouse/issues/21437). [#21438](https://github.com/ClickHouse/ClickHouse/pull/21438) ([Ilya Golshtein](https://github.com/ilejn)). +* Fixed race on SSL object inside SecureSocket in Poco. [#21456](https://github.com/ClickHouse/ClickHouse/pull/21456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix that S3 table holds old credentials after config update. [#21457](https://github.com/ClickHouse/ClickHouse/pull/21457) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). +* Fix table function `clusterAllReplicas` returns wrong `_shard_num`. close [#21481](https://github.com/ClickHouse/ClickHouse/issues/21481). [#21498](https://github.com/ClickHouse/ClickHouse/pull/21498) ([flynn](https://github.com/ucasfl)). +* The ```::poll()``` return ```rc == 1 ```, it could be a request or it could be a response. [#21544](https://github.com/ClickHouse/ClickHouse/pull/21544) ([小路](https://github.com/nicelulu)). +* In case if query has constant `WHERE` condition, and setting `optimize_skip_unused_shards` enabled, all shards may be skipped and query could return incorrect empty result. [#21550](https://github.com/ClickHouse/ClickHouse/pull/21550) ([Amos Bird](https://github.com/amosbird)). +* Fix possible error ` Cannot find column` when `optimize_skip_unused_shards` is enabled and zero shards are used. [#21579](https://github.com/ClickHouse/ClickHouse/pull/21579) ([Azat Khuzhin](https://github.com/azat)). +* `std::terminate` was called if there is an error writing data into s3. [#21624](https://github.com/ClickHouse/ClickHouse/pull/21624) ([Vladimir C](https://github.com/vdimir)). +* Remove unknown columns from joined table in where for queries to external database engines (MySQL, PostgreSQL). close [#14614](https://github.com/ClickHouse/ClickHouse/issues/14614), close [#19288](https://github.com/ClickHouse/ClickHouse/issues/19288) (dup), close [#19645](https://github.com/ClickHouse/ClickHouse/issues/19645) (dup). [#21640](https://github.com/ClickHouse/ClickHouse/pull/21640) ([Vladimir C](https://github.com/vdimir)). +* Fix fsync_part_directory for horizontal merge. [#21642](https://github.com/ClickHouse/ClickHouse/pull/21642) ([Azat Khuzhin](https://github.com/azat)). +* Fix distributed requests cancellation (for example simple select from multiple shards with limit, i.e. `select * from remote('127.{2,3}', system.numbers) limit 100`) with `async_socket_for_remote=1`. [#21643](https://github.com/ClickHouse/ClickHouse/pull/21643) ([Azat Khuzhin](https://github.com/azat)). +* Add type conversion for StorageJoin (previously led to SIGSEGV). [#21646](https://github.com/ClickHouse/ClickHouse/pull/21646) ([Azat Khuzhin](https://github.com/azat)). +* Start accepting connections after DDLWorker and dictionaries initialization. [#21676](https://github.com/ClickHouse/ClickHouse/pull/21676) ([Azat Khuzhin](https://github.com/azat)). +* Fix SIGSEGV on not existing attributes from ip_trie with access_to_key_from_attributes. [#21692](https://github.com/ClickHouse/ClickHouse/pull/21692) ([Azat Khuzhin](https://github.com/azat)). +* Fix function `arrayElement` with type `Map` for constant integer arguments. [#21699](https://github.com/ClickHouse/ClickHouse/pull/21699) ([Anton Popov](https://github.com/CurtizJ)). +* Fix concurrent `OPTIMIZE` and `DROP` for `ReplicatedMergeTree`. [#21716](https://github.com/ClickHouse/ClickHouse/pull/21716) ([Azat Khuzhin](https://github.com/azat)). +* Fix bug for ReplicatedMerge table engines when `ALTER MODIFY COLUMN` query doesn't change the type of decimal column if its size (32 bit or 64 bit) doesn't change. [#21728](https://github.com/ClickHouse/ClickHouse/pull/21728) ([alesapin](https://github.com/alesapin)). +* Reverted S3 connection pools. [#21737](https://github.com/ClickHouse/ClickHouse/pull/21737) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix adding of parts with already existing in destination table names in query `MOVE PARTITION TO TABLE` with non-replicated `MergeTree` tables. [#21760](https://github.com/ClickHouse/ClickHouse/pull/21760) ([ygrek](https://github.com/ygrek)). +* Fix scalar subquery index analysis. This fixes [#21717](https://github.com/ClickHouse/ClickHouse/issues/21717) , which was introduced in https://github.com/ClickHouse/ClickHouse/pull/18896 . [#21766](https://github.com/ClickHouse/ClickHouse/pull/21766) ([Amos Bird](https://github.com/amosbird)). +* Fix possible crashes in aggregate functions with combinator Distinct, while using two-level aggregation. This is a follow-up fix of https://github.com/ClickHouse/ClickHouse/pull/18365 . Can only reproduced in production env. No test case available yet. cc @CurtizJ. [#21818](https://github.com/ClickHouse/ClickHouse/pull/21818) ([Amos Bird](https://github.com/amosbird)). +* Better error handling and logging in WriteBufferFromS3. [#21836](https://github.com/ClickHouse/ClickHouse/pull/21836) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix incorrect query result (and possible crash) which could happen when `WHERE` or `HAVING` condition is pushed before `GROUP BY`. Fixes [#21773](https://github.com/ClickHouse/ClickHouse/issues/21773). [#21841](https://github.com/ClickHouse/ClickHouse/pull/21841) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix deadlock in first catboost model execution. Closes [#13832](https://github.com/ClickHouse/ClickHouse/issues/13832). [#21844](https://github.com/ClickHouse/ClickHouse/pull/21844) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix wrong `ORDER BY` results when a query contains window functions, and optimization for reading in primary key order is applied. Fixes [#21828](https://github.com/ClickHouse/ClickHouse/issues/21828). [#21915](https://github.com/ClickHouse/ClickHouse/pull/21915) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix reading the HTTP POST request with "multipart/form-data" content type. [#21936](https://github.com/ClickHouse/ClickHouse/pull/21936) ([Ivan](https://github.com/abyss7)). +* Prevent hedged connections overlaps (`Unknown packet 9 from server` error). [#21941](https://github.com/ClickHouse/ClickHouse/pull/21941) ([Azat Khuzhin](https://github.com/azat)). +* Reverted [#15454](https://github.com/ClickHouse/ClickHouse/issues/15454) that may cause significant increase in memory usage while loading external dictionaries of hashed type. This closes [#21935](https://github.com/ClickHouse/ClickHouse/issues/21935). [#21948](https://github.com/ClickHouse/ClickHouse/pull/21948) ([Maksim Kita](https://github.com/kitaisreal)). +* In rare case, merge for `CollapsingMergeTree` may create granule with `index_granularity + 1` rows. Because of this, internal check, added in [#18928](https://github.com/ClickHouse/ClickHouse/issues/18928) (affects 21.2 and 21.3), may fail with error `Incomplete granules are not allowed while blocks are granules size`. This error did not allow parts to merge. [#21976](https://github.com/ClickHouse/ClickHouse/pull/21976) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* The function `decrypt` was lacking a check for the minimal size of data encrypted in AEAD mode. This closes [#21897](https://github.com/ClickHouse/ClickHouse/issues/21897). [#22064](https://github.com/ClickHouse/ClickHouse/pull/22064) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Docker entrypoint: avoid chown of `.` in case when `LOG_PATH` is empty. Closes [#22100](https://github.com/ClickHouse/ClickHouse/issues/22100). [#22102](https://github.com/ClickHouse/ClickHouse/pull/22102) ([filimonov](https://github.com/filimonov)). +* Disable `async_socket_for_remote`/`use_hedged_requests` for buggy linux kernels. [#22109](https://github.com/ClickHouse/ClickHouse/pull/22109) ([Azat Khuzhin](https://github.com/azat)). +* Fix waiting for `OPTIMIZE` and `ALTER` queries for `ReplicatedMergeTree` table engines. Now the query will not hang when the table was detached or restarted. [#22118](https://github.com/ClickHouse/ClickHouse/pull/22118) ([alesapin](https://github.com/alesapin)). +* Fix the background thread pool name. [#22122](https://github.com/ClickHouse/ClickHouse/pull/22122) ([fastio](https://github.com/fastio)). +* Fix error `Invalid number of rows in Chunk` in `JOIN` with `TOTALS` and `arrayJoin`. Closes [#19303](https://github.com/ClickHouse/ClickHouse/issues/19303). [#22129](https://github.com/ClickHouse/ClickHouse/pull/22129) ([Vladimir C](https://github.com/vdimir)). +* Fix docker entrypoint in case `http_port` is not in the config. [#22132](https://github.com/ClickHouse/ClickHouse/pull/22132) ([Ewout](https://github.com/devwout)). +* Fix uncaught exception in InterserverIOHTTPHandler. [#22146](https://github.com/ClickHouse/ClickHouse/pull/22146) ([Azat Khuzhin](https://github.com/azat)). +* Use finalize() over next() for nested writers. [#22147](https://github.com/ClickHouse/ClickHouse/pull/22147) ([Azat Khuzhin](https://github.com/azat)). +* Fix query cancellation with `use_hedged_requests=0` and `async_socket_for_remote=1`. [#22183](https://github.com/ClickHouse/ClickHouse/pull/22183) ([Azat Khuzhin](https://github.com/azat)). +* Fix exception which may happen when `SELECT` has constant `WHERE` condition and source table has columns which names are digits. [#22270](https://github.com/ClickHouse/ClickHouse/pull/22270) ([LiuNeng](https://github.com/liuneng1994)). +* Now clickhouse will not throw `LOGICAL_ERROR` exception when we try to mutate the already covered part. Fixes [#22013](https://github.com/ClickHouse/ClickHouse/issues/22013). [#22291](https://github.com/ClickHouse/ClickHouse/pull/22291) ([alesapin](https://github.com/alesapin)). +* Fixed bug in S3 zero-copy replication for hybrid storage. [#22378](https://github.com/ClickHouse/ClickHouse/pull/22378) ([ianton-ru](https://github.com/ianton-ru)). +* Add (missing) memory accounting in parallel parsing routines. In previous versions OOM was possible when the resultset contains very large blocks of data. This closes [#22008](https://github.com/ClickHouse/ClickHouse/issues/22008). [#22425](https://github.com/ClickHouse/ClickHouse/pull/22425) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove socket from epoll before cancelling packet receiver in HedgedConnections to prevent possible race. I hope it fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)). + +#### Build/Testing/Packaging Improvement +* Fix macOS shared lib build. [#20184](https://github.com/ClickHouse/ClickHouse/pull/20184) ([nvartolomei](https://github.com/nvartolomei)). +* - Added ALL and NONE privilege tests. - Added ROW POLICY tests. - Cleanup of existing tests. - Tests close faster if something fails. [#21354](https://github.com/ClickHouse/ClickHouse/pull/21354) ([MyroTk](https://github.com/MyroTk)). +* Fixing LDAP authentication performance test by removing assertion. [#21507](https://github.com/ClickHouse/ClickHouse/pull/21507) ([vzakaznikov](https://github.com/vzakaznikov)). +* Updating docker/test/testflows/runner/dockerd-entrypoint.sh to use Yandex dockerhub-proxy. [#21551](https://github.com/ClickHouse/ClickHouse/pull/21551) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add llvm-12 binaries name to search in cmake scripts. Implicit constants conversions to mute clang warnings. Updated submodules to build with CMake 3.19. Mute recursion in macro expansion in readpassphrase library. Deprecated -fuse-ld changed to --ld-path for clang. [#21597](https://github.com/ClickHouse/ClickHouse/pull/21597) ([Ilya Yatsishin](https://github.com/qoega)). +* Updating TestFlows to 1.6.74. [#21673](https://github.com/ClickHouse/ClickHouse/pull/21673) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add [jepsen](https://github.com/jepsen-io/jepsen) tests for NuKeeper. [#21677](https://github.com/ClickHouse/ClickHouse/pull/21677) ([alesapin](https://github.com/alesapin)). +* remove decode method with python3. [#21832](https://github.com/ClickHouse/ClickHouse/pull/21832) ([kevin wan](https://github.com/MaxWk)). +* Allow to use clang-tidy with release builds by enabling assertions if it is used. [#21914](https://github.com/ClickHouse/ClickHouse/pull/21914) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Introduce 2 arguments for clickhouse-server image Dockerfile: deb_location & single_binary_location. [#21977](https://github.com/ClickHouse/ClickHouse/pull/21977) ([filimonov](https://github.com/filimonov)). +* Add `tzdata` to Docker containers because reading ORC formats requires it. This closes [#14156](https://github.com/ClickHouse/ClickHouse/issues/14156). [#22000](https://github.com/ClickHouse/ClickHouse/pull/22000) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable status check for SQLancer CI run. [#22015](https://github.com/ClickHouse/ClickHouse/pull/22015) ([Ilya Yatsishin](https://github.com/qoega)). +* Run stateless tests in parallel in CI. Depends on [#22181](https://github.com/ClickHouse/ClickHouse/issues/22181). [#22300](https://github.com/ClickHouse/ClickHouse/pull/22300) ([alesapin](https://github.com/alesapin)). +* try fix [#22289](https://github.com/ClickHouse/ClickHouse/issues/22289) https://clickhouse-test-reports.s3.yandex.net/22289/c71da4a5c8e655f4bdfaa33b92ab022b97dfdf1a/integration_tests_(asan).html#fail1 MySQL is started only once with MaterializeMySQL integration test. [#22341](https://github.com/ClickHouse/ClickHouse/pull/22341) ([Winter Zhang](https://github.com/zhang2014)). +* - Added a way to check memory info for the RBAC testflows tests. [#22403](https://github.com/ClickHouse/ClickHouse/pull/22403) ([MyroTk](https://github.com/MyroTk)). +* Fixed compiling on ppc64le and use the correct instruction pointer register on ppc64le. [#22430](https://github.com/ClickHouse/ClickHouse/pull/22430) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix ClickHouse's config embedding and cctz's timezone embedding on ppc64le. ... [#22445](https://github.com/ClickHouse/ClickHouse/pull/22445) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Enable building with jemalloc on ppc64le ... [#22447](https://github.com/ClickHouse/ClickHouse/pull/22447) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix Fedora\RHEL\CentOS not finding libclang_rt.builtins on ppc64le ... [#22458](https://github.com/ClickHouse/ClickHouse/pull/22458) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix CMake error about internal CMake variable CMAKE_ASM_COMPILE_OBJECT not set on ppc64le ... [#22469](https://github.com/ClickHouse/ClickHouse/pull/22469) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix compiling boost on ppc64le ... [#22474](https://github.com/ClickHouse/ClickHouse/pull/22474) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Enable compiling on ppc64le with Clang ... [#22476](https://github.com/ClickHouse/ClickHouse/pull/22476) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Re-enable the S3 (AWS) library on aarch64 ... [#22484](https://github.com/ClickHouse/ClickHouse/pull/22484) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Enable the bundled openldap on ppc64le ... [#22487](https://github.com/ClickHouse/ClickHouse/pull/22487) ([Kfir Itzhak](https://github.com/mastertheknife)). + +#### Other +* Update tests for hedged requests. [#21998](https://github.com/ClickHouse/ClickHouse/pull/21998) ([Kruglov Pavel](https://github.com/Avogar)). +* Don't set the same timeouts in ReadBufferFromPocoSocket/WriteBufferFromPocoSocket in nextImpl because it causes a race. [#22343](https://github.com/ClickHouse/ClickHouse/pull/22343) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Improve the translation of `query_log.md` in Chinese documents'. [#21729](https://github.com/ClickHouse/ClickHouse/pull/21729) ([Pysaoke](https://github.com/baixuexue123)). +* NO CL ENTRY: 'Update gui.md: add SeekTable'. [#21768](https://github.com/ClickHouse/ClickHouse/pull/21768) ([Vitaliy Fedorchenko](https://github.com/VitaliyMF)). +* NO CL ENTRY: 'Flatten libcpuid PEERDIRs'. [#22078](https://github.com/ClickHouse/ClickHouse/pull/22078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* NO CL ENTRY: 'Revert "quick fix for broken resolution of apt.llvm.org on Yandex infra"'. [#22374](https://github.com/ClickHouse/ClickHouse/pull/22374) ([alesapin](https://github.com/alesapin)). + +#### New Feature (datasketches support in clickhouse #14893) + +* Support ThetaSketch to do set operations. [#22207](https://github.com/ClickHouse/ClickHouse/pull/22207) ([Ping Yu](https://github.com/pingyu)). + diff --git a/docs/changelogs/v21.4.2.10-prestable.md b/docs/changelogs/v21.4.2.10-prestable.md new file mode 100644 index 00000000000..1bc440d126c --- /dev/null +++ b/docs/changelogs/v21.4.2.10-prestable.md @@ -0,0 +1,176 @@ +### ClickHouse release v21.4.2.10-prestable FIXME as compared to v21.3.1.6185-prestable + +#### Backward Incompatible Change +* Column `keys` in table `system.dictionaries` was replaced to columns `key.names` and `key.types`. Columns `key.names`, `key.types`, `attribute.names`, `attribute.types` from `system.dictionaries` table does not require dictionary to be loaded. [#21884](https://github.com/ClickHouse/ClickHouse/pull/21884) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix `cutToFirstSignificantSubdomainCustom()`/`firstSignificantSubdomainCustom()` returning wrong result for 3+ level domains present in custom top-level domain list. For input domains matching these custom top-level domains, the third-level domain was considered to be the first significant one. This is now fixed. This change may introduce incompatibility if the function is used in e.g. the sharding key. [#21946](https://github.com/ClickHouse/ClickHouse/pull/21946) ([Azat Khuzhin](https://github.com/azat)). +* The `toStartOfIntervalFunction` will align hour intervals to the midnight (in previous versions they were aligned to the start of unix epoch). For example, `toStartOfInterval(x, INTERVAL 11 HOUR)` will split every day into three intervals: 00:00:00..10:59:59, 11:00:00..21:59:59 and 22:00:00..23:59:59. This behaviour is more suited for practical needs. This closes [#9510](https://github.com/ClickHouse/ClickHouse/issues/9510). [#22060](https://github.com/ClickHouse/ClickHouse/pull/22060) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* Extended range of `DateTime64` to properly support dates from year 1925 to 2283. Improved support of `DateTime` around zero date (`1970-01-01`). ... [#9404](https://github.com/ClickHouse/ClickHouse/pull/9404) ([Vasily Nemkov](https://github.com/Enmk)). +* - Added support of Kerberos authentication for preconfigured users and HTTP requests (GSS-SPNEGO). [#14995](https://github.com/ClickHouse/ClickHouse/pull/14995) ([Denis Glazachev](https://github.com/traceon)). +* Zero-copy replication for ReplicatedMergeTree over S3 storage. [#16240](https://github.com/ClickHouse/ClickHouse/pull/16240) ([ianton-ru](https://github.com/ianton-ru)). +* Support `dictHas` function for `RangeHashedDictionary`. Fixes [#6680](https://github.com/ClickHouse/ClickHouse/issues/6680). [#19816](https://github.com/ClickHouse/ClickHouse/pull/19816) ([Maksim Kita](https://github.com/kitaisreal)). +* Supports implicit key type conversion for JOIN. Closes [#18567](https://github.com/ClickHouse/ClickHouse/issues/18567). [#19885](https://github.com/ClickHouse/ClickHouse/pull/19885) ([Vladimir C](https://github.com/vdimir)). +* Allow customizing timeouts for http connections used for replication independently from other http timeouts. [#20088](https://github.com/ClickHouse/ClickHouse/pull/20088) ([nvartolomei](https://github.com/nvartolomei)). +* Added async update in `ComplexKeyCache`, `SSDCache`, `SSDComplexKeyCache` dictionaries. Added support for Nullable type in `Cache`, `ComplexKeyCache`, `SSDCache`, `SSDComplexKeyCache` dictionaries. Added support for multiple attributes fetch with `dictGet`, `dictGetOrDefault` functions. Fixes [#21517](https://github.com/ClickHouse/ClickHouse/issues/21517). [#20595](https://github.com/ClickHouse/ClickHouse/pull/20595) ([Maksim Kita](https://github.com/kitaisreal)). +* Added `Grant,` `Revoke` and `System` values of `query_kind` column for corresponding queries in `system.query_log` ... [#21102](https://github.com/ClickHouse/ClickHouse/pull/21102) ([Vasily Nemkov](https://github.com/Enmk)). +* Added new SQL command ALTER TABLE 'table_name' UNFREEZE [PARTITION 'part_expr'] WITH NAME 'backup_name'. [#21142](https://github.com/ClickHouse/ClickHouse/pull/21142) ([Pavel Kovalenko](https://github.com/Jokser)). +* Added ExecutablePool dictionary source. Close [#14528](https://github.com/ClickHouse/ClickHouse/issues/14528). [#21321](https://github.com/ClickHouse/ClickHouse/pull/21321) ([Maksim Kita](https://github.com/kitaisreal)). +* - Add function `isIPAddressInRange` to test if an IPv4 or IPv6 address is contained in a given CIDR network prefix. [#21329](https://github.com/ClickHouse/ClickHouse/pull/21329) ([PHO](https://github.com/depressed-pho)). +* Add `_partition_id` virtual column for `MergeTree*` engines. Allow to prune partitions by `_partition_id`. Add `partitionID()` function to calculate partition id string. [#21401](https://github.com/ClickHouse/ClickHouse/pull/21401) ([Amos Bird](https://github.com/amosbird)). +* Add new column `slowdowns_count` to `system.clusters`. When using hedged requests, it shows how many times we switched to another replica because this replica was responding slowly. Also show actual value of `errors_count` in `system.clusters`. [#21480](https://github.com/ClickHouse/ClickHouse/pull/21480) ([Kruglov Pavel](https://github.com/Avogar)). +* Add option `--backslash` for clickhouse-format, which can add a backslash at the end of each line of the formatted query. [#21494](https://github.com/ClickHouse/ClickHouse/pull/21494) ([flynn](https://github.com/ucasfl)). +* Add new optional clause GRANTEES for CREATE/ALTER USER commands:. [#21641](https://github.com/ClickHouse/ClickHouse/pull/21641) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add `ctime` option to `zookeeper-dump-tree`. It allows to dump node creation time. [#21842](https://github.com/ClickHouse/ClickHouse/pull/21842) ([Ilya](https://github.com/HumanUser)). +* Functions 'dictGet', 'dictHas' use current database name if it is not specified for dictionaries created with DDL. Closes [#21632](https://github.com/ClickHouse/ClickHouse/issues/21632). [#21859](https://github.com/ClickHouse/ClickHouse/pull/21859) ([Maksim Kita](https://github.com/kitaisreal)). +* Support `Nullable` type for `PolygonDictionary` attribute. [#21890](https://github.com/ClickHouse/ClickHouse/pull/21890) ([Maksim Kita](https://github.com/kitaisreal)). +* Added table function `dictionary`. It works the same way as `Dictionary` engine. Closes [#21560](https://github.com/ClickHouse/ClickHouse/issues/21560). [#21910](https://github.com/ClickHouse/ClickHouse/pull/21910) ([Maksim Kita](https://github.com/kitaisreal)). +* Add function `timezoneOf` that returns the timezone name of `DateTime` or `DateTime64` data types. This does not close [#9959](https://github.com/ClickHouse/ClickHouse/issues/9959). Fix inconsistencies in function names: add aliases `timezone` and `timeZone` as well as `toTimezone` and `toTimeZone` and `timezoneOf` and `timeZoneOf`. [#22001](https://github.com/ClickHouse/ClickHouse/pull/22001) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `prefer_column_name_to_alias` setting to use original column names instead of aliases. it is needed to be more compatible with common databases' aliasing rules. This is for [#9715](https://github.com/ClickHouse/ClickHouse/issues/9715) and [#9887](https://github.com/ClickHouse/ClickHouse/issues/9887). [#22044](https://github.com/ClickHouse/ClickHouse/pull/22044) ([Amos Bird](https://github.com/amosbird)). + +#### Performance Improvement +* Support parallel parsing for `CSVWithNames` and `TSVWithNames` formats. This closes [#21085](https://github.com/ClickHouse/ClickHouse/issues/21085). [#21149](https://github.com/ClickHouse/ClickHouse/pull/21149) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Improved performance by replacing `memcpy` to another implementation. This closes [#18583](https://github.com/ClickHouse/ClickHouse/issues/18583). [#21520](https://github.com/ClickHouse/ClickHouse/pull/21520) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Supported parallel formatting in clickhouse-local and everywhere else. [#21630](https://github.com/ClickHouse/ClickHouse/pull/21630) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Optimize performance of queries like `SELECT ... FINAL ... WHERE`. Now in queries with `FINAL` it's allowed to move to `PREWHERE` columns, which are in sorting key. ... [#21830](https://github.com/ClickHouse/ClickHouse/pull/21830) ([foolchi](https://github.com/foolchi)). +* Faster `GROUP BY` with small `max_rows_to_group_by` and `group_by_overflow_mode='any'`. [#21856](https://github.com/ClickHouse/ClickHouse/pull/21856) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Avoid unnecessary data copy when using codec `NONE`. Please note that codec `NONE` is mostly useless - it's recommended to always use compression (`LZ4` is by default). Despite the common belief, disabling compression may not improve performance (the opposite effect is possible). The `NONE` codec is useful in some cases: - when data is uncompressable; - for synthetic benchmarks. [#22145](https://github.com/ClickHouse/ClickHouse/pull/22145) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add cache for files read with `min_bytes_to_use_mmap_io` setting. It makes significant (2x and more) performance improvement when the value of the setting is small by avoiding frequent mmap/munmap calls and the consequent page faults. Note that mmap IO has major drawbacks that makes it less reliable in production (e.g. hung or SIGBUS on faulty disks; less controllable memory usage). Nevertheless it is good in benchmarks. [#22206](https://github.com/ClickHouse/ClickHouse/pull/22206) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable read with mmap IO for file ranges from 64 MiB (the settings `min_bytes_to_use_mmap_io`). It may lead to moderate performance improvement. [#22326](https://github.com/ClickHouse/ClickHouse/pull/22326) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Introduce a new merge tree setting `min_bytes_to_rebalance_partition_over_jbod` which allows assigning new parts to different disks of a JBOD volume in a balanced way. [#16481](https://github.com/ClickHouse/ClickHouse/pull/16481) ([Amos Bird](https://github.com/amosbird)). +* Improve performance of aggregation in order of sorting key (with enabled setting `optimize_aggregation_in_order`). [#19401](https://github.com/ClickHouse/ClickHouse/pull/19401) ([Anton Popov](https://github.com/CurtizJ)). +* MaterializeMySQL: add minmax skipping index for _version column. [#20382](https://github.com/ClickHouse/ClickHouse/pull/20382) ([Stig Bakken](https://github.com/stigsb)). +* Do not create empty parts on INSERT when `optimize_on_insert` setting enabled. Fixes [#20304](https://github.com/ClickHouse/ClickHouse/issues/20304). [#20387](https://github.com/ClickHouse/ClickHouse/pull/20387) ([Kruglov Pavel](https://github.com/Avogar)). +* - Support more cases to rewrite `CROSS JOIN` to `INNER JOIN`. [#20392](https://github.com/ClickHouse/ClickHouse/pull/20392) ([Vladimir C](https://github.com/vdimir)). +* MaterializeMySQL: Attempt to reconnect to MySQL if the connection is lost. [#20961](https://github.com/ClickHouse/ClickHouse/pull/20961) ([Håvard Kvålen](https://github.com/havardk)). +* Improve support of integer keys in data type `Map`. [#21157](https://github.com/ClickHouse/ClickHouse/pull/21157) ([Anton Popov](https://github.com/CurtizJ)). +* Improve clickhouse-format to not throw exception when there are extra spaces or comment after the last query, and throw exception early with readable message when format `ASTInsertQuery` with data . [#21311](https://github.com/ClickHouse/ClickHouse/pull/21311) ([flynn](https://github.com/ucasfl)). +* Age and Precision in graphite rollup configs should increase from retention to retention. Now it's checked and the wrong config raises an exception. [#21496](https://github.com/ClickHouse/ClickHouse/pull/21496) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add setting `optimize_skip_unused_shards_limit` to limit the number of sharding key values for `optimize_skip_unused_shards`. [#21512](https://github.com/ClickHouse/ClickHouse/pull/21512) ([Azat Khuzhin](https://github.com/azat)). +* Add `last_error_time`/`last_error_message`/`last_error_stacktrace`/`remote` columns for `system.errors`. [#21529](https://github.com/ClickHouse/ClickHouse/pull/21529) ([Azat Khuzhin](https://github.com/azat)). +* If PODArray was instantiated with element size that is neither a fraction or a multiple of 16, buffer overflow was possible. No bugs in current releases exist. [#21533](https://github.com/ClickHouse/ClickHouse/pull/21533) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* - Propagate query and session settings for distributed DDL queries. Set `distributed_ddl_entry_format_version` to 2 to enable this. - Added `distributed_ddl_output_mode` setting. Supported modes: `none`, `throw` (default), `null_status_on_timeout` and `never_throw`. - Miscellaneous fixes and improvements for `Replicated` database engine. [#21535](https://github.com/ClickHouse/ClickHouse/pull/21535) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update clusters only if their configurations were updated. [#21685](https://github.com/ClickHouse/ClickHouse/pull/21685) ([Kruglov Pavel](https://github.com/Avogar)). +* Support replicas priority for postgres dictionary source. [#21710](https://github.com/ClickHouse/ClickHouse/pull/21710) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Closes [#21701](https://github.com/ClickHouse/ClickHouse/issues/21701). Support non-default table schema for postgres storage/table-function. [#21711](https://github.com/ClickHouse/ClickHouse/pull/21711) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better formatting for `Array` and `Map` data types in Web UI. [#21798](https://github.com/ClickHouse/ClickHouse/pull/21798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* DiskS3 (experimental feature under development). Fixed bug with the impossibility to move directory if the destination is not empty and cache disk is used. [#21837](https://github.com/ClickHouse/ClickHouse/pull/21837) ([Pavel Kovalenko](https://github.com/Jokser)). +* Add connection pool for PostgreSQL table/database engine and dictionary source. Should fix [#21444](https://github.com/ClickHouse/ClickHouse/issues/21444). [#21839](https://github.com/ClickHouse/ClickHouse/pull/21839) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add profile event HedgedRequestsChangeReplica, change read data timeout from sec to ms. [#21886](https://github.com/ClickHouse/ClickHouse/pull/21886) ([Kruglov Pavel](https://github.com/Avogar)). +* Support `RANGE OFFSET` frame for floating point types. Implement `lagInFrame`/`leadInFrame` window functions, which are analogous to `lag`/`lead`, but respect the window frame. They are identical when the frame is `between unbounded preceding and unbounded following`. This closes [#5485](https://github.com/ClickHouse/ClickHouse/issues/5485). [#21895](https://github.com/ClickHouse/ClickHouse/pull/21895) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Show path to data directory of `EmbeddedRocksDB` tables in system tables. [#21903](https://github.com/ClickHouse/ClickHouse/pull/21903) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Supported `replication_alter_partitions_sync=1` setting for moving partitions from helping table to destination. Decreased default timeouts. Fixes [#21911](https://github.com/ClickHouse/ClickHouse/issues/21911). [#21912](https://github.com/ClickHouse/ClickHouse/pull/21912) ([jasong](https://github.com/songenjie)). +* If partition key of a `MergeTree` table does not include `Date` or `DateTime` columns but includes exactly one `DateTime64` column, expose its values in the `min_time` and `max_time` columns in `system.parts` and `system.parts_columns` tables. Add `min_time` and `max_time` columns to `system.parts_columns` table (these was inconsistency to the `system.parts` table). This closes [#18244](https://github.com/ClickHouse/ClickHouse/issues/18244). [#22011](https://github.com/ClickHouse/ClickHouse/pull/22011) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* * Add option strict_increase to windowFunnel function to calculate each event once (resolve [#21835](https://github.com/ClickHouse/ClickHouse/issues/21835)). [#22025](https://github.com/ClickHouse/ClickHouse/pull/22025) ([Vladimir C](https://github.com/vdimir)). +* Added case insensitive aliases for `CONNECTION_ID()` and `VERSION()` functions. This fixes [#22028](https://github.com/ClickHouse/ClickHouse/issues/22028). [#22042](https://github.com/ClickHouse/ClickHouse/pull/22042) ([Eugene Klimov](https://github.com/Slach)). +* Update used version of simdjson to 0.9.1. This fixes [#21984](https://github.com/ClickHouse/ClickHouse/issues/21984). [#22057](https://github.com/ClickHouse/ClickHouse/pull/22057) ([Vitaly Baranov](https://github.com/vitlibar)). +* Convert `system.errors.stack_trace` from `String` into `Array(UInt64)` (This should decrease overhead for the errors collecting). [#22058](https://github.com/ClickHouse/ClickHouse/pull/22058) ([Azat Khuzhin](https://github.com/azat)). +* If tuple of NULLs, e.g. `(NULL, NULL)` is on the left hand side of `IN` operator with tuples of non-NULLs on the right hand side, e.g. `SELECT (NULL, NULL) IN ((0, 0), (3, 1))` return 0 instead of throwing an exception about incompatible types. The expression may also appear due to optimization of something like `SELECT (NULL, NULL) = (8, 0) OR (NULL, NULL) = (3, 2) OR (NULL, NULL) = (0, 0) OR (NULL, NULL) = (3, 1)`. This closes [#22017](https://github.com/ClickHouse/ClickHouse/issues/22017). [#22063](https://github.com/ClickHouse/ClickHouse/pull/22063) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added possibility to migrate existing S3 disk to the schema with backup-restore capabilities. [#22070](https://github.com/ClickHouse/ClickHouse/pull/22070) ([Pavel Kovalenko](https://github.com/Jokser)). +* Add case-insensitive history search/navigation and subword movement features to clickhouse-client. [#22105](https://github.com/ClickHouse/ClickHouse/pull/22105) ([Amos Bird](https://github.com/amosbird)). +* Add `current_database` column to `system.processes` table. It contains the current database of the query. [#22365](https://github.com/ClickHouse/ClickHouse/pull/22365) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix MSan report for function `range` with `UInt256` argument (support for large integers is experimental). This closes [#22157](https://github.com/ClickHouse/ClickHouse/issues/22157). [#22387](https://github.com/ClickHouse/ClickHouse/pull/22387) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error `Directory tmp_fetch_XXX already exists` which could happen after failed fetch part. Delete temporary fetch directory if it already exists. Fixes [#14197](https://github.com/ClickHouse/ClickHouse/issues/14197). [#22411](https://github.com/ClickHouse/ClickHouse/pull/22411) ([nvartolomei](https://github.com/nvartolomei)). +* Better exception message in client in case of exception while server is writing blocks. In previous versions client may get misleading message like `Data compressed with different methods`. [#22427](https://github.com/ClickHouse/ClickHouse/pull/22427) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Fixed open behavior of remote host filter in case when there is `remote_url_allow_hosts` section in configuration but no entries there. :warning: please add a note about potential issue when upgrading - @alexey-milovidov. [#20058](https://github.com/ClickHouse/ClickHouse/pull/20058) ([Vladimir Chebotarev](https://github.com/excitoon)). +* `force_drop_table` flag didn't work for `MATERIALIZED VIEW`, it's fixed. Fixes [#18943](https://github.com/ClickHouse/ClickHouse/issues/18943). [#20626](https://github.com/ClickHouse/ClickHouse/pull/20626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix official website documents which introduced cluster secret feature. [#21331](https://github.com/ClickHouse/ClickHouse/pull/21331) ([Chao Ma](https://github.com/godliness)). +* Fix receive and send timeouts and non-blocking read in secure socket. [#21429](https://github.com/ClickHouse/ClickHouse/pull/21429) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix Avro format parsing for Kafka. Fixes [#21437](https://github.com/ClickHouse/ClickHouse/issues/21437). [#21438](https://github.com/ClickHouse/ClickHouse/pull/21438) ([Ilya Golshtein](https://github.com/ilejn)). +* Fixed race on SSL object inside SecureSocket in Poco. [#21456](https://github.com/ClickHouse/ClickHouse/pull/21456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix that S3 table holds old credentials after config update. [#21457](https://github.com/ClickHouse/ClickHouse/pull/21457) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). +* Fix table function `clusterAllReplicas` returns wrong `_shard_num`. close [#21481](https://github.com/ClickHouse/ClickHouse/issues/21481). [#21498](https://github.com/ClickHouse/ClickHouse/pull/21498) ([flynn](https://github.com/ucasfl)). +* The ```::poll()``` return ```rc == 1 ```, it could be a request or it could be a response. [#21544](https://github.com/ClickHouse/ClickHouse/pull/21544) ([小路](https://github.com/nicelulu)). +* In case if query has constant `WHERE` condition, and setting `optimize_skip_unused_shards` enabled, all shards may be skipped and query could return incorrect empty result. [#21550](https://github.com/ClickHouse/ClickHouse/pull/21550) ([Amos Bird](https://github.com/amosbird)). +* Fix possible error ` Cannot find column` when `optimize_skip_unused_shards` is enabled and zero shards are used. [#21579](https://github.com/ClickHouse/ClickHouse/pull/21579) ([Azat Khuzhin](https://github.com/azat)). +* `std::terminate` was called if there is an error writing data into s3. [#21624](https://github.com/ClickHouse/ClickHouse/pull/21624) ([Vladimir C](https://github.com/vdimir)). +* Remove unknown columns from joined table in where for queries to external database engines (MySQL, PostgreSQL). close [#14614](https://github.com/ClickHouse/ClickHouse/issues/14614), close [#19288](https://github.com/ClickHouse/ClickHouse/issues/19288) (dup), close [#19645](https://github.com/ClickHouse/ClickHouse/issues/19645) (dup). [#21640](https://github.com/ClickHouse/ClickHouse/pull/21640) ([Vladimir C](https://github.com/vdimir)). +* Fix fsync_part_directory for horizontal merge. [#21642](https://github.com/ClickHouse/ClickHouse/pull/21642) ([Azat Khuzhin](https://github.com/azat)). +* Fix distributed requests cancellation (for example simple select from multiple shards with limit, i.e. `select * from remote('127.{2,3}', system.numbers) limit 100`) with `async_socket_for_remote=1`. [#21643](https://github.com/ClickHouse/ClickHouse/pull/21643) ([Azat Khuzhin](https://github.com/azat)). +* Add type conversion for StorageJoin (previously led to SIGSEGV). [#21646](https://github.com/ClickHouse/ClickHouse/pull/21646) ([Azat Khuzhin](https://github.com/azat)). +* Start accepting connections after DDLWorker and dictionaries initialization. [#21676](https://github.com/ClickHouse/ClickHouse/pull/21676) ([Azat Khuzhin](https://github.com/azat)). +* Fix SIGSEGV on not existing attributes from ip_trie with access_to_key_from_attributes. [#21692](https://github.com/ClickHouse/ClickHouse/pull/21692) ([Azat Khuzhin](https://github.com/azat)). +* Fix function `arrayElement` with type `Map` for constant integer arguments. [#21699](https://github.com/ClickHouse/ClickHouse/pull/21699) ([Anton Popov](https://github.com/CurtizJ)). +* Fix concurrent `OPTIMIZE` and `DROP` for `ReplicatedMergeTree`. [#21716](https://github.com/ClickHouse/ClickHouse/pull/21716) ([Azat Khuzhin](https://github.com/azat)). +* Fix bug for ReplicatedMerge table engines when `ALTER MODIFY COLUMN` query doesn't change the type of decimal column if its size (32 bit or 64 bit) doesn't change. [#21728](https://github.com/ClickHouse/ClickHouse/pull/21728) ([alesapin](https://github.com/alesapin)). +* Reverted S3 connection pools. [#21737](https://github.com/ClickHouse/ClickHouse/pull/21737) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix adding of parts with already existing in destination table names in query `MOVE PARTITION TO TABLE` with non-replicated `MergeTree` tables. [#21760](https://github.com/ClickHouse/ClickHouse/pull/21760) ([ygrek](https://github.com/ygrek)). +* Fix scalar subquery index analysis. This fixes [#21717](https://github.com/ClickHouse/ClickHouse/issues/21717) , which was introduced in https://github.com/ClickHouse/ClickHouse/pull/18896 . [#21766](https://github.com/ClickHouse/ClickHouse/pull/21766) ([Amos Bird](https://github.com/amosbird)). +* Fix possible crashes in aggregate functions with combinator Distinct, while using two-level aggregation. This is a follow-up fix of https://github.com/ClickHouse/ClickHouse/pull/18365 . Can only reproduced in production env. No test case available yet. cc @CurtizJ. [#21818](https://github.com/ClickHouse/ClickHouse/pull/21818) ([Amos Bird](https://github.com/amosbird)). +* Better error handling and logging in WriteBufferFromS3. [#21836](https://github.com/ClickHouse/ClickHouse/pull/21836) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fix incorrect query result (and possible crash) which could happen when `WHERE` or `HAVING` condition is pushed before `GROUP BY`. Fixes [#21773](https://github.com/ClickHouse/ClickHouse/issues/21773). [#21841](https://github.com/ClickHouse/ClickHouse/pull/21841) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix deadlock in first catboost model execution. Closes [#13832](https://github.com/ClickHouse/ClickHouse/issues/13832). [#21844](https://github.com/ClickHouse/ClickHouse/pull/21844) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#22625](https://github.com/ClickHouse/ClickHouse/issues/22625): Fix bug, which leads to underaggregation of data in case of enabled `optimize_aggregation_in_order` and many parts in table. Slightly improve performance of aggregation with enabled `optimize_aggregation_in_order`. [#21889](https://github.com/ClickHouse/ClickHouse/pull/21889) ([Anton Popov](https://github.com/CurtizJ)). +* Fix wrong `ORDER BY` results when a query contains window functions, and optimization for reading in primary key order is applied. Fixes [#21828](https://github.com/ClickHouse/ClickHouse/issues/21828). [#21915](https://github.com/ClickHouse/ClickHouse/pull/21915) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix reading the HTTP POST request with "multipart/form-data" content type. [#21936](https://github.com/ClickHouse/ClickHouse/pull/21936) ([Ivan](https://github.com/abyss7)). +* Prevent hedged connections overlaps (`Unknown packet 9 from server` error). [#21941](https://github.com/ClickHouse/ClickHouse/pull/21941) ([Azat Khuzhin](https://github.com/azat)). +* Reverted [#15454](https://github.com/ClickHouse/ClickHouse/issues/15454) that may cause significant increase in memory usage while loading external dictionaries of hashed type. This closes [#21935](https://github.com/ClickHouse/ClickHouse/issues/21935). [#21948](https://github.com/ClickHouse/ClickHouse/pull/21948) ([Maksim Kita](https://github.com/kitaisreal)). +* In rare case, merge for `CollapsingMergeTree` may create granule with `index_granularity + 1` rows. Because of this, internal check, added in [#18928](https://github.com/ClickHouse/ClickHouse/issues/18928) (affects 21.2 and 21.3), may fail with error `Incomplete granules are not allowed while blocks are granules size`. This error did not allow parts to merge. [#21976](https://github.com/ClickHouse/ClickHouse/pull/21976) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* The function `decrypt` was lacking a check for the minimal size of data encrypted in AEAD mode. This closes [#21897](https://github.com/ClickHouse/ClickHouse/issues/21897). [#22064](https://github.com/ClickHouse/ClickHouse/pull/22064) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Docker entrypoint: avoid chown of `.` in case when `LOG_PATH` is empty. Closes [#22100](https://github.com/ClickHouse/ClickHouse/issues/22100). [#22102](https://github.com/ClickHouse/ClickHouse/pull/22102) ([filimonov](https://github.com/filimonov)). +* Disable `async_socket_for_remote`/`use_hedged_requests` for buggy linux kernels. [#22109](https://github.com/ClickHouse/ClickHouse/pull/22109) ([Azat Khuzhin](https://github.com/azat)). +* Fix waiting for `OPTIMIZE` and `ALTER` queries for `ReplicatedMergeTree` table engines. Now the query will not hang when the table was detached or restarted. [#22118](https://github.com/ClickHouse/ClickHouse/pull/22118) ([alesapin](https://github.com/alesapin)). +* Fix the background thread pool name. [#22122](https://github.com/ClickHouse/ClickHouse/pull/22122) ([fastio](https://github.com/fastio)). +* Fix error `Invalid number of rows in Chunk` in `JOIN` with `TOTALS` and `arrayJoin`. Closes [#19303](https://github.com/ClickHouse/ClickHouse/issues/19303). [#22129](https://github.com/ClickHouse/ClickHouse/pull/22129) ([Vladimir C](https://github.com/vdimir)). +* Fix docker entrypoint in case `http_port` is not in the config. [#22132](https://github.com/ClickHouse/ClickHouse/pull/22132) ([Ewout](https://github.com/devwout)). +* Fix uncaught exception in InterserverIOHTTPHandler. [#22146](https://github.com/ClickHouse/ClickHouse/pull/22146) ([Azat Khuzhin](https://github.com/azat)). +* Use finalize() over next() for nested writers. [#22147](https://github.com/ClickHouse/ClickHouse/pull/22147) ([Azat Khuzhin](https://github.com/azat)). +* Fix query cancellation with `use_hedged_requests=0` and `async_socket_for_remote=1`. [#22183](https://github.com/ClickHouse/ClickHouse/pull/22183) ([Azat Khuzhin](https://github.com/azat)). +* Fix exception which may happen when `SELECT` has constant `WHERE` condition and source table has columns which names are digits. [#22270](https://github.com/ClickHouse/ClickHouse/pull/22270) ([LiuNeng](https://github.com/liuneng1994)). +* Now clickhouse will not throw `LOGICAL_ERROR` exception when we try to mutate the already covered part. Fixes [#22013](https://github.com/ClickHouse/ClickHouse/issues/22013). [#22291](https://github.com/ClickHouse/ClickHouse/pull/22291) ([alesapin](https://github.com/alesapin)). +* Backported in [#22541](https://github.com/ClickHouse/ClickHouse/issues/22541): Do not limit HTTP chunk size. Fixes [#21907](https://github.com/ClickHouse/ClickHouse/issues/21907). [#22322](https://github.com/ClickHouse/ClickHouse/pull/22322) ([Ivan](https://github.com/abyss7)). +* Fixed bug in S3 zero-copy replication for hybrid storage. [#22378](https://github.com/ClickHouse/ClickHouse/pull/22378) ([ianton-ru](https://github.com/ianton-ru)). +* Backported in [#22532](https://github.com/ClickHouse/ClickHouse/issues/22532): Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add (missing) memory accounting in parallel parsing routines. In previous versions OOM was possible when the resultset contains very large blocks of data. This closes [#22008](https://github.com/ClickHouse/ClickHouse/issues/22008). [#22425](https://github.com/ClickHouse/ClickHouse/pull/22425) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22519](https://github.com/ClickHouse/ClickHouse/issues/22519): Remove socket from epoll before cancelling packet receiver in HedgedConnections to prevent possible race. I hope it fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#22617](https://github.com/ClickHouse/ClickHouse/issues/22617): Fix ClickHouseDictionarySource configuration loop. Closes [#14314](https://github.com/ClickHouse/ClickHouse/issues/14314). [#22479](https://github.com/ClickHouse/ClickHouse/pull/22479) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#22558](https://github.com/ClickHouse/ClickHouse/issues/22558): Fix bug in partial merge join with `LowCardinality`. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#22559](https://github.com/ClickHouse/ClickHouse/issues/22559): Fix deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#22577](https://github.com/ClickHouse/ClickHouse/issues/22577): Fix UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22649](https://github.com/ClickHouse/ClickHouse/issues/22649): Avoid UB in *Log engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22652](https://github.com/ClickHouse/ClickHouse/issues/22652): Try flush write buffer only if it is initialized. Fixes segfault when client closes connection very early [#22579](https://github.com/ClickHouse/ClickHouse/issues/22579). [#22591](https://github.com/ClickHouse/ClickHouse/pull/22591) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#22680](https://github.com/ClickHouse/ClickHouse/issues/22680): Fix LOGICAL_ERROR for Log with nested types w/o columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)). + +#### Build/Testing/Packaging Improvement +* Fix macOS shared lib build. [#20184](https://github.com/ClickHouse/ClickHouse/pull/20184) ([nvartolomei](https://github.com/nvartolomei)). +* - Added ALL and NONE privilege tests. - Added ROW POLICY tests. - Cleanup of existing tests. - Tests close faster if something fails. [#21354](https://github.com/ClickHouse/ClickHouse/pull/21354) ([MyroTk](https://github.com/MyroTk)). +* Fixing LDAP authentication performance test by removing assertion. [#21507](https://github.com/ClickHouse/ClickHouse/pull/21507) ([vzakaznikov](https://github.com/vzakaznikov)). +* Updating docker/test/testflows/runner/dockerd-entrypoint.sh to use Yandex dockerhub-proxy. [#21551](https://github.com/ClickHouse/ClickHouse/pull/21551) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add llvm-12 binaries name to search in cmake scripts. Implicit constants conversions to mute clang warnings. Updated submodules to build with CMake 3.19. Mute recursion in macro expansion in readpassphrase library. Deprecated -fuse-ld changed to --ld-path for clang. [#21597](https://github.com/ClickHouse/ClickHouse/pull/21597) ([Ilya Yatsishin](https://github.com/qoega)). +* Updating TestFlows to 1.6.74. [#21673](https://github.com/ClickHouse/ClickHouse/pull/21673) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add [jepsen](https://github.com/jepsen-io/jepsen) tests for NuKeeper. [#21677](https://github.com/ClickHouse/ClickHouse/pull/21677) ([alesapin](https://github.com/alesapin)). +* remove decode method with python3. [#21832](https://github.com/ClickHouse/ClickHouse/pull/21832) ([kevin wan](https://github.com/MaxWk)). +* Allow to use clang-tidy with release builds by enabling assertions if it is used. [#21914](https://github.com/ClickHouse/ClickHouse/pull/21914) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Introduce 2 arguments for clickhouse-server image Dockerfile: deb_location & single_binary_location. [#21977](https://github.com/ClickHouse/ClickHouse/pull/21977) ([filimonov](https://github.com/filimonov)). +* Add `tzdata` to Docker containers because reading ORC formats requires it. This closes [#14156](https://github.com/ClickHouse/ClickHouse/issues/14156). [#22000](https://github.com/ClickHouse/ClickHouse/pull/22000) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable status check for SQLancer CI run. [#22015](https://github.com/ClickHouse/ClickHouse/pull/22015) ([Ilya Yatsishin](https://github.com/qoega)). +* try fix [#22289](https://github.com/ClickHouse/ClickHouse/issues/22289) https://clickhouse-test-reports.s3.yandex.net/22289/c71da4a5c8e655f4bdfaa33b92ab022b97dfdf1a/integration_tests_(asan).html#fail1 MySQL is started only once with MaterializeMySQL integration test. [#22341](https://github.com/ClickHouse/ClickHouse/pull/22341) ([Winter Zhang](https://github.com/zhang2014)). +* - Added a way to check memory info for the RBAC testflows tests. [#22403](https://github.com/ClickHouse/ClickHouse/pull/22403) ([MyroTk](https://github.com/MyroTk)). +* Fixed compiling on ppc64le and use the correct instruction pointer register on ppc64le. [#22430](https://github.com/ClickHouse/ClickHouse/pull/22430) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix ClickHouse's config embedding and cctz's timezone embedding on ppc64le. ... [#22445](https://github.com/ClickHouse/ClickHouse/pull/22445) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Enable building with jemalloc on ppc64le ... [#22447](https://github.com/ClickHouse/ClickHouse/pull/22447) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix Fedora\RHEL\CentOS not finding libclang_rt.builtins on ppc64le ... [#22458](https://github.com/ClickHouse/ClickHouse/pull/22458) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix CMake error about internal CMake variable CMAKE_ASM_COMPILE_OBJECT not set on ppc64le ... [#22469](https://github.com/ClickHouse/ClickHouse/pull/22469) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix compiling boost on ppc64le ... [#22474](https://github.com/ClickHouse/ClickHouse/pull/22474) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Enable compiling on ppc64le with Clang ... [#22476](https://github.com/ClickHouse/ClickHouse/pull/22476) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Re-enable the S3 (AWS) library on aarch64 ... [#22484](https://github.com/ClickHouse/ClickHouse/pull/22484) ([Kfir Itzhak](https://github.com/mastertheknife)). + +#### Other +* Update tests for hedged requests. [#21998](https://github.com/ClickHouse/ClickHouse/pull/21998) ([Kruglov Pavel](https://github.com/Avogar)). +* Don't set the same timeouts in ReadBufferFromPocoSocket/WriteBufferFromPocoSocket in nextImpl because it causes a race. [#22343](https://github.com/ClickHouse/ClickHouse/pull/22343) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Improve the translation of `query_log.md` in Chinese documents'. [#21729](https://github.com/ClickHouse/ClickHouse/pull/21729) ([Pysaoke](https://github.com/baixuexue123)). +* NO CL ENTRY: 'Update gui.md: add SeekTable'. [#21768](https://github.com/ClickHouse/ClickHouse/pull/21768) ([Vitaliy Fedorchenko](https://github.com/VitaliyMF)). +* NO CL ENTRY: 'Flatten libcpuid PEERDIRs'. [#22078](https://github.com/ClickHouse/ClickHouse/pull/22078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* NO CL ENTRY: 'Revert "quick fix for broken resolution of apt.llvm.org on Yandex infra"'. [#22374](https://github.com/ClickHouse/ClickHouse/pull/22374) ([alesapin](https://github.com/alesapin)). + +#### New Feature (datasketches support in clickhouse #14893) + +* Support ThetaSketch to do set operations. [#22207](https://github.com/ClickHouse/ClickHouse/pull/22207) ([Ping Yu](https://github.com/pingyu)). + diff --git a/docs/changelogs/v21.4.3.21-stable.md b/docs/changelogs/v21.4.3.21-stable.md new file mode 100644 index 00000000000..dc3d7b7005b --- /dev/null +++ b/docs/changelogs/v21.4.3.21-stable.md @@ -0,0 +1,18 @@ +### ClickHouse release v21.4.3.21-stable FIXME as compared to v21.4.2.10-prestable + +#### Improvement +* Backported in [#22934](https://github.com/ClickHouse/ClickHouse/issues/22934): Correctly check structure of async distributed blocks. [#22325](https://github.com/ClickHouse/ClickHouse/pull/22325) ([Azat Khuzhin](https://github.com/azat)). + +#### Bug Fix +* Backported in [#22966](https://github.com/ClickHouse/ClickHouse/issues/22966): Fix very rare bug when quorum insert with `quorum_parallel=1` is not really "quorum" because of deduplication. [#18215](https://github.com/ClickHouse/ClickHouse/pull/18215) ([filimonov](https://github.com/filimonov)). +* Backported in [#22722](https://github.com/ClickHouse/ClickHouse/issues/22722): Check if table function view is used as a column. This complements https://github.com/ClickHouse/ClickHouse/pull/20350. [#21465](https://github.com/ClickHouse/ClickHouse/pull/21465) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#22808](https://github.com/ClickHouse/ClickHouse/issues/22808): Follow-up fix for [#21936](https://github.com/ClickHouse/ClickHouse/issues/21936). Also fixes [#22433](https://github.com/ClickHouse/ClickHouse/issues/22433). [#22518](https://github.com/ClickHouse/ClickHouse/pull/22518) ([Ivan](https://github.com/abyss7)). +* Backported in [#22757](https://github.com/ClickHouse/ClickHouse/issues/22757): Fix usage of function `map` in distributed queries. [#22588](https://github.com/ClickHouse/ClickHouse/pull/22588) ([foolchi](https://github.com/foolchi)). +* Backported in [#22702](https://github.com/ClickHouse/ClickHouse/issues/22702): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). +* Backported in [#22741](https://github.com/ClickHouse/ClickHouse/issues/22741): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#22888](https://github.com/ClickHouse/ClickHouse/issues/22888): Fix approx total rows accounting for reverse reading from MergeTree. [#22726](https://github.com/ClickHouse/ClickHouse/pull/22726) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#22884](https://github.com/ClickHouse/ClickHouse/issues/22884): Fix pushdown of `HAVING` in case, when filter column is used in aggregation. [#22763](https://github.com/ClickHouse/ClickHouse/pull/22763) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#22917](https://github.com/ClickHouse/ClickHouse/issues/22917): LIVE VIEW (experimental feature). Fix possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, see https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e. [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#22921](https://github.com/ClickHouse/ClickHouse/issues/22921): Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#22957](https://github.com/ClickHouse/ClickHouse/issues/22957): Fix usage of constant columns of type `Map` with nullable values. [#22939](https://github.com/ClickHouse/ClickHouse/pull/22939) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.4.4.30-stable.md b/docs/changelogs/v21.4.4.30-stable.md new file mode 100644 index 00000000000..f029d334fc7 --- /dev/null +++ b/docs/changelogs/v21.4.4.30-stable.md @@ -0,0 +1,25 @@ +### ClickHouse release v21.4.4.30-stable FIXME as compared to v21.4.3.21-stable + +#### Backward Incompatible Change +* Backported in [#23145](https://github.com/ClickHouse/ClickHouse/issues/23145): Now replicas that are processing the `ALTER TABLE ATTACH PART[ITION]` command search in their `detached/` folders before fetching the data from other replicas. As an implementation detail, a new command `ATTACH_PART` is introduced in the replicated log. Parts are searched and compared by their checksums. [#18978](https://github.com/ClickHouse/ClickHouse/pull/18978) ([Mike Kot](https://github.com/myrrc)). + +#### New Feature +* Backported in [#23147](https://github.com/ClickHouse/ClickHouse/issues/23147): Improved performance of `dictGetHierarchy`, `dictIsIn` functions. Added functions `dictGetChildren(dictionary, key)`, `dictGetDescendants(dictionary, key, level)`. Function `dictGetChildren` return all children as an array if indexes. It is a inverse transformation for `dictGetHierarchy`. Function `dictGetDescendants` return all descendants as if `dictGetChildren` was applied `level` times recursively. Zero `level` value is equivalent to infinity. Closes [#14656](https://github.com/ClickHouse/ClickHouse/issues/14656). [#22096](https://github.com/ClickHouse/ClickHouse/pull/22096) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23146](https://github.com/ClickHouse/ClickHouse/issues/23146): Added function `dictGetOrNull`. It works like `dictGet`, but return `Null` in case key was not found in dictionary. Closes [#22375](https://github.com/ClickHouse/ClickHouse/issues/22375). [#22413](https://github.com/ClickHouse/ClickHouse/pull/22413) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Improvement +* Backported in [#23148](https://github.com/ClickHouse/ClickHouse/issues/23148): Add aliases `simpleJSONExtract/simpleJSONHas` to `visitParam/visitParamExtract{UInt, Int, Bool, Float, Raw, String}`. Fixes [#21383](https://github.com/ClickHouse/ClickHouse/issues/21383). [#21519](https://github.com/ClickHouse/ClickHouse/pull/21519) ([fastio](https://github.com/fastio)). +* Backported in [#23016](https://github.com/ClickHouse/ClickHouse/issues/23016): Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23079](https://github.com/ClickHouse/ClickHouse/issues/23079): Raised the threshold on max number of matches in result of the function `extractAllGroupsHorizontal`. [#23036](https://github.com/ClickHouse/ClickHouse/pull/23036) ([Vasily Nemkov](https://github.com/Enmk)). + +#### Bug Fix +* Backported in [#23158](https://github.com/ClickHouse/ClickHouse/issues/23158): Fixed a bug with unlimited wait for auxiliary AWS requests. [#22594](https://github.com/ClickHouse/ClickHouse/pull/22594) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#23018](https://github.com/ClickHouse/ClickHouse/issues/23018): fixed `formatDateTime()` on `DateTime64` and "%C" format specifier fixed `toDateTime64()` for large values and non-zero scale. ... [#22937](https://github.com/ClickHouse/ClickHouse/pull/22937) ([Vasily Nemkov](https://github.com/Enmk)). +* Backported in [#23031](https://github.com/ClickHouse/ClickHouse/issues/23031): Fix error `Cannot find column in ActionsDAG result` which may happen if subquery uses `untuple`. Fixes [#22290](https://github.com/ClickHouse/ClickHouse/issues/22290). [#22991](https://github.com/ClickHouse/ClickHouse/pull/22991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#23074](https://github.com/ClickHouse/ClickHouse/issues/23074): Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23097](https://github.com/ClickHouse/ClickHouse/issues/23097): Fixed `Table .inner_id... doesn't exist` error when selecting from Materialized View after detaching it from Atomic database and attaching back. [#23047](https://github.com/ClickHouse/ClickHouse/pull/23047) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23173](https://github.com/ClickHouse/ClickHouse/issues/23173): Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Build/Testing/Packaging Improvement +* Backported in [#23149](https://github.com/ClickHouse/ClickHouse/issues/23149): Enable the bundled openldap on ppc64le ... [#22487](https://github.com/ClickHouse/ClickHouse/pull/22487) ([Kfir Itzhak](https://github.com/mastertheknife)). + diff --git a/docs/changelogs/v21.4.5.46-stable.md b/docs/changelogs/v21.4.5.46-stable.md new file mode 100644 index 00000000000..664037ba596 --- /dev/null +++ b/docs/changelogs/v21.4.5.46-stable.md @@ -0,0 +1,21 @@ +### ClickHouse release v21.4.5.46-stable FIXME as compared to v21.4.4.30-stable + +#### Improvement +* Backported in [#23273](https://github.com/ClickHouse/ClickHouse/issues/23273): Disable settings `use_hedged_requests` and `async_socket_for_remote` because there is an evidence that it may cause issues. [#23261](https://github.com/ClickHouse/ClickHouse/pull/23261) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#23234](https://github.com/ClickHouse/ClickHouse/issues/23234): Server might fail to start if `data_type_default_nullable` setting is enabled in default profile, it's fixed. Fixes [#22573](https://github.com/ClickHouse/ClickHouse/issues/22573). [#23185](https://github.com/ClickHouse/ClickHouse/pull/23185) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23407](https://github.com/ClickHouse/ClickHouse/issues/23407): QueryAliasVisitor to prefer alias for ASTWithAlias if subquery was optimized to constant. Fixes [#22924](https://github.com/ClickHouse/ClickHouse/issues/22924). Fixes [#10401](https://github.com/ClickHouse/ClickHouse/issues/10401). [#23191](https://github.com/ClickHouse/ClickHouse/pull/23191) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23382](https://github.com/ClickHouse/ClickHouse/issues/23382): Fixed `Not found column` error when selecting from `MaterializeMySQL` with condition on key column. Fixes [#22432](https://github.com/ClickHouse/ClickHouse/issues/22432). [#23200](https://github.com/ClickHouse/ClickHouse/pull/23200) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23385](https://github.com/ClickHouse/ClickHouse/issues/23385): Fixed the behavior when disabling `input_format_with_names_use_header ` setting discards all the input with CSVWithNames format. This fixes [#22406](https://github.com/ClickHouse/ClickHouse/issues/22406). [#23202](https://github.com/ClickHouse/ClickHouse/pull/23202) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#23287](https://github.com/ClickHouse/ClickHouse/issues/23287): Fixed simple key dictionary from DDL creation if primary key is not first attribute. Fixes [#23236](https://github.com/ClickHouse/ClickHouse/issues/23236). [#23262](https://github.com/ClickHouse/ClickHouse/pull/23262) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23451](https://github.com/ClickHouse/ClickHouse/issues/23451): Fixed very rare (distributed) race condition between creation and removal of ReplicatedMergeTree tables. It might cause exceptions like `node doesn't exist` on attempt to create replicated table. Fixes [#21419](https://github.com/ClickHouse/ClickHouse/issues/21419). [#23294](https://github.com/ClickHouse/ClickHouse/pull/23294) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23426](https://github.com/ClickHouse/ClickHouse/issues/23426): Fix possible crash in case if `unknown packet` was received form remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#23467](https://github.com/ClickHouse/ClickHouse/issues/23467): `ORDER BY` with `COLLATE` was not working correctly if the column is in primary key (or is a monotonic function of it) and the setting `optimize_read_in_order` is not turned off. This closes [#22379](https://github.com/ClickHouse/ClickHouse/issues/22379). Workaround for older versions: turn the setting `optimize_read_in_order` off. [#23375](https://github.com/ClickHouse/ClickHouse/pull/23375) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23509](https://github.com/ClickHouse/ClickHouse/issues/23509): Remove support for `argMin` and `argMax` for single `Tuple` argument. The code was not memory-safe. The feature was added by mistake and it is confusing for people. These functions can be reintroduced under different names later. This fixes [#22384](https://github.com/ClickHouse/ClickHouse/issues/22384) and reverts [#17359](https://github.com/ClickHouse/ClickHouse/issues/17359). [#23393](https://github.com/ClickHouse/ClickHouse/pull/23393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23478](https://github.com/ClickHouse/ClickHouse/issues/23478): Kafka storage may support `arrow` and `arrowstream` format messages. [#23415](https://github.com/ClickHouse/ClickHouse/pull/23415) ([Chao Ma](https://github.com/godliness)). +* Backported in [#23498](https://github.com/ClickHouse/ClickHouse/issues/23498): - Bug fix for `deltaSum` aggregate function in counter reset case ... [#23437](https://github.com/ClickHouse/ClickHouse/pull/23437) ([Russ Frank](https://github.com/rf)). +* Backported in [#23492](https://github.com/ClickHouse/ClickHouse/issues/23492): Fix bug that does not allow cast from empty array literal, to array with dimensions greater than 1. Closes [#14476](https://github.com/ClickHouse/ClickHouse/issues/14476). [#23456](https://github.com/ClickHouse/ClickHouse/pull/23456) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23534](https://github.com/ClickHouse/ClickHouse/issues/23534): When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#23531](https://github.com/ClickHouse/ClickHouse/issues/23531): Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.4.6.55-stable.md b/docs/changelogs/v21.4.6.55-stable.md new file mode 100644 index 00000000000..ea3e413ea0c --- /dev/null +++ b/docs/changelogs/v21.4.6.55-stable.md @@ -0,0 +1,18 @@ +### ClickHouse release v21.4.6.55-stable FIXME as compared to v21.4.5.46-stable + +#### Improvement +* Backported in [#23679](https://github.com/ClickHouse/ClickHouse/issues/23679): Fixed `quantile(s)TDigest`. Added special handling of singleton centroids according to tdunning/t-digest 3.2+. Also a bug with over-compression of centroids in implementation of earlier version of the algorithm was fixed. [#23314](https://github.com/ClickHouse/ClickHouse/pull/23314) ([Vladimir Chebotarev](https://github.com/excitoon)). + +#### Bug Fix +* Backported in [#23580](https://github.com/ClickHouse/ClickHouse/issues/23580): Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23676](https://github.com/ClickHouse/ClickHouse/issues/23676): Don't relax NOT conditions during partition pruning. This fixes [#23305](https://github.com/ClickHouse/ClickHouse/issues/23305) and [#21539](https://github.com/ClickHouse/ClickHouse/issues/21539). [#23310](https://github.com/ClickHouse/ClickHouse/pull/23310) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#23583](https://github.com/ClickHouse/ClickHouse/issues/23583): * Fix bug in dict join with join_algorithm = 'auto'. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#23641](https://github.com/ClickHouse/ClickHouse/issues/23641): Fix incompatible constant expression generation during partition pruning based on virtual columns. This fixes https://github.com/ClickHouse/ClickHouse/pull/21401#discussion_r611888913. [#23366](https://github.com/ClickHouse/ClickHouse/pull/23366) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#23586](https://github.com/ClickHouse/ClickHouse/issues/23586): Allow to move more conditions to `PREWHERE` as it was before version 21.1. Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#23588](https://github.com/ClickHouse/ClickHouse/issues/23588): Fixed `Cannot unlink file` error on unsuccessful creation of ReplicatedMergeTree table with multidisk configuration. This closes [#21755](https://github.com/ClickHouse/ClickHouse/issues/21755). [#23433](https://github.com/ClickHouse/ClickHouse/pull/23433) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23591](https://github.com/ClickHouse/ClickHouse/issues/23591): Fix corner cases in vertical merges with `ReplacingMergeTree`. In rare cases they could lead to fails of merges with exceptions like `Incomplete granules are not allowed while blocks are granules size`. [#23459](https://github.com/ClickHouse/ClickHouse/pull/23459) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#23611](https://github.com/ClickHouse/ClickHouse/issues/23611): Fix restart / stop command hanging. Closes [#20214](https://github.com/ClickHouse/ClickHouse/issues/20214). [#23552](https://github.com/ClickHouse/ClickHouse/pull/23552) ([filimonov](https://github.com/filimonov)). +* Backported in [#23693](https://github.com/ClickHouse/ClickHouse/issues/23693): Fixed server fault when inserting data through HTTP caused an exception. This fixes [#23512](https://github.com/ClickHouse/ClickHouse/issues/23512). [#23643](https://github.com/ClickHouse/ClickHouse/pull/23643) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#23695](https://github.com/ClickHouse/ClickHouse/issues/23695): Added an exception in case of completely the same values in both samples in aggregate function `mannWhitneyUTest`. This fixes [#23646](https://github.com/ClickHouse/ClickHouse/issues/23646). [#23654](https://github.com/ClickHouse/ClickHouse/pull/23654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#23774](https://github.com/ClickHouse/ClickHouse/issues/23774): Avoid possible "Cannot schedule a task" error (in case some exception had been occurred) on INSERT into Distributed. [#23744](https://github.com/ClickHouse/ClickHouse/pull/23744) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.4.7.3-stable.md b/docs/changelogs/v21.4.7.3-stable.md new file mode 100644 index 00000000000..0dad6cfcb2b --- /dev/null +++ b/docs/changelogs/v21.4.7.3-stable.md @@ -0,0 +1,17 @@ +### ClickHouse release v21.4.7.3-stable FIXME as compared to v21.4.6.55-stable + +#### Bug Fix +* Backported in [#23971](https://github.com/ClickHouse/ClickHouse/issues/23971): Fixed a bug in recovery of staled `ReplicatedMergeTree` replica. Some metadata updates could be ignored by staled replica if `ALTER` query was executed during downtime of the replica. [#23742](https://github.com/ClickHouse/ClickHouse/pull/23742) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23819](https://github.com/ClickHouse/ClickHouse/issues/23819): Fix crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#23816](https://github.com/ClickHouse/ClickHouse/issues/23816): Fix `CLEAR COLUMN` does not work when it is referenced by materialized view. Close [#23764](https://github.com/ClickHouse/ClickHouse/issues/23764). [#23781](https://github.com/ClickHouse/ClickHouse/pull/23781) ([flynn](https://github.com/ucasfl)). +* Backported in [#23831](https://github.com/ClickHouse/ClickHouse/issues/23831): Fix error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` and `use_hedged_requests`. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#23930](https://github.com/ClickHouse/ClickHouse/issues/23930): HashedDictionary complex key update field initial load fix. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23957](https://github.com/ClickHouse/ClickHouse/issues/23957): Fix keys metrics accounting for CACHE() dictionary with duplicates in the source (leads to `DictCacheKeysRequestedMiss` overflows). [#23929](https://github.com/ClickHouse/ClickHouse/pull/23929) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#24003](https://github.com/ClickHouse/ClickHouse/issues/24003): Fix SIGSEGV for external GROUP BY and overflow row (i.e. queries like `SELECT FROM GROUP BY WITH TOTALS SETTINGS max_bytes_before_external_group_by>0, max_rows_to_group_by>0, group_by_overflow_mode='any', totals_mode='before_having'`). [#23962](https://github.com/ClickHouse/ClickHouse/pull/23962) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#24188](https://github.com/ClickHouse/ClickHouse/issues/24188): Some `ALTER PARTITION` queries might cause `Part A intersects previous part B` and `Unexpected merged part C intersecting drop range D` errors in replication queue. It's fixed. Fixes [#23296](https://github.com/ClickHouse/ClickHouse/issues/23296). [#23997](https://github.com/ClickHouse/ClickHouse/pull/23997) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24031](https://github.com/ClickHouse/ClickHouse/issues/24031): Fix crash in MergeJoin, close [#24010](https://github.com/ClickHouse/ClickHouse/issues/24010). [#24013](https://github.com/ClickHouse/ClickHouse/pull/24013) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#24141](https://github.com/ClickHouse/ClickHouse/issues/24141): Bug: explain pipeline with` select xxx final `shows wrong pipeline: ``` dell123 :) explain pipeline select z from prewhere_move_select_final final;. [#24116](https://github.com/ClickHouse/ClickHouse/pull/24116) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#24171](https://github.com/ClickHouse/ClickHouse/issues/24171): Fix a rare bug that could lead to a partially initialized table that can serve write requests (insert/alter/so on). Now such tables will be in readonly mode. [#24122](https://github.com/ClickHouse/ClickHouse/pull/24122) ([alesapin](https://github.com/alesapin)). +* Backported in [#24215](https://github.com/ClickHouse/ClickHouse/issues/24215): Fix race condition which could happen in RBAC under a heavy load. This PR fixes [#24090](https://github.com/ClickHouse/ClickHouse/issues/24090), [#24134](https://github.com/ClickHouse/ClickHouse/issues/24134),. [#24176](https://github.com/ClickHouse/ClickHouse/pull/24176) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#24243](https://github.com/ClickHouse/ClickHouse/issues/24243): Fix abnormal server termination due to hdfs becoming not accessible during query execution. Closes [#24117](https://github.com/ClickHouse/ClickHouse/issues/24117). [#24191](https://github.com/ClickHouse/ClickHouse/pull/24191) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.5.1.6601-prestable.md b/docs/changelogs/v21.5.1.6601-prestable.md new file mode 100644 index 00000000000..2d09ce6c20b --- /dev/null +++ b/docs/changelogs/v21.5.1.6601-prestable.md @@ -0,0 +1,109 @@ +### ClickHouse release v21.5.1.6601-prestable FIXME as compared to v21.4.1.6422-prestable + +#### Backward Incompatible Change +* Change comparison of integers and floating point numbers when integer is not exactly representable in the floating point data type. In new version comparison will return false as the rounding error will occur. Example: `9223372036854775808.0 != 9223372036854775808`, because the number `9223372036854775808` is not representable as floating point number exactly (and `9223372036854775808.0` is rounded to `9223372036854776000.0`). But in previous version the comparison will return as the numbers are equal, because if the floating point number `9223372036854776000.0` get converted back to UInt64, it will yield `9223372036854775808`. For the reference, the Python programming language also treats these numbers as equal. But this behaviour was dependend on CPU model (different results on AMD64 and AArch64 for some out-of-range numbers), so we make the comparison more precise. It will treat int and float numbers equal only if int is represented in floating point type exactly. [#22595](https://github.com/ClickHouse/ClickHouse/pull/22595) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* Implement function `arrayFold(x1,...,xn,accum -> expression, array1,...,arrayn, init_accum)` that applies the expression to each element of the array (or set of parallel arrays) and collect result in accumulator. [#21589](https://github.com/ClickHouse/ClickHouse/pull/21589) ([Dmitry Krylov](https://github.com/dmalkr)). +* - Support Apple m1. [#21639](https://github.com/ClickHouse/ClickHouse/pull/21639) ([changvvb](https://github.com/changvvb)). +* Add a setting `max_distributed_depth` that limits the depth of recursive queries to `Distributed` tables. Closes [#20229](https://github.com/ClickHouse/ClickHouse/issues/20229). [#21942](https://github.com/ClickHouse/ClickHouse/pull/21942) ([flynn](https://github.com/ucasfl)). +* Table function, which allows to process files from `s3` in parallel from many nodes in a specified cluster. [#22012](https://github.com/ClickHouse/ClickHouse/pull/22012) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Support for replicas in MySQL/PostgreSQL table engine / table function. Added wrapper storage over MySQL / PostgreSQL storages to allow shards. Closes [#20969](https://github.com/ClickHouse/ClickHouse/issues/20969). [#22217](https://github.com/ClickHouse/ClickHouse/pull/22217) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update paths to the catboost model configs in config reloading. [#22434](https://github.com/ClickHouse/ClickHouse/pull/22434) ([Kruglov Pavel](https://github.com/Avogar)). +* Add new setting `non_replicated_deduplication_window` for non-replicated MergeTree inserts deduplication. [#22514](https://github.com/ClickHouse/ClickHouse/pull/22514) ([alesapin](https://github.com/alesapin)). +* FlatDictionary added `initial_array_size`, `max_array_size` options. [#22521](https://github.com/ClickHouse/ClickHouse/pull/22521) ([Maksim Kita](https://github.com/kitaisreal)). +* Added `ALTER TABLE ... FETCH PART ...` query. It's similar to `FETCH PARTITION`, but fetches only one part. [#22706](https://github.com/ClickHouse/ClickHouse/pull/22706) ([jasong](https://github.com/songenjie)). +* Added `Decimal256` type support in dictionaries. Closes [#20979](https://github.com/ClickHouse/ClickHouse/issues/20979). [#22960](https://github.com/ClickHouse/ClickHouse/pull/22960) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Performance Improvement +* Add function alignment for possibly better performance. [#21431](https://github.com/ClickHouse/ClickHouse/pull/21431) ([Daniel Kutenin](https://github.com/danlark1)). +* Exclude values that does not belong to the shard from right part of IN section for distributed queries (under `optimize_skip_unused_shards_rewrite_in`, enabled by default, since it still requires `optimize_skip_unused_shards`). [#21511](https://github.com/ClickHouse/ClickHouse/pull/21511) ([Azat Khuzhin](https://github.com/azat)). +* Disable compression by default when interacting with localhost (with clickhouse-client or server to server with distributed queries) via native protocol. It may improve performance of some import/export operations. This closes [#22234](https://github.com/ClickHouse/ClickHouse/issues/22234). [#22237](https://github.com/ClickHouse/ClickHouse/pull/22237) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve performance of reading from `ArrowStream` input format for sources other then local file (e.g. URL). [#22673](https://github.com/ClickHouse/ClickHouse/pull/22673) ([nvartolomei](https://github.com/nvartolomei)). +* Improve performance of `intDiv` by dynamic dispatch for AVX2. This closes [#22314](https://github.com/ClickHouse/ClickHouse/issues/22314). [#23000](https://github.com/ClickHouse/ClickHouse/pull/23000) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Support dynamic interserver credentials. [#14113](https://github.com/ClickHouse/ClickHouse/pull/14113) ([johnskopis](https://github.com/johnskopis)). +* Add clickhouse-library-bridge for library dictionary source. Closes [#9502](https://github.com/ClickHouse/ClickHouse/issues/9502). [#21509](https://github.com/ClickHouse/ClickHouse/pull/21509) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Allow publishing Kafka errors to a virtual column of Kafka engine, controlled by the `kafka_handle_error_mode` setting. [#21850](https://github.com/ClickHouse/ClickHouse/pull/21850) ([fastio](https://github.com/fastio)). +* Use nanodbc instead of Poco::ODBC. Closes [#9678](https://github.com/ClickHouse/ClickHouse/issues/9678). Add support for DateTime64 and Decimal* for ODBC table engine. Closes [#21961](https://github.com/ClickHouse/ClickHouse/issues/21961). Fixed issue with cyrillic text being truncated. Closes [#16246](https://github.com/ClickHouse/ClickHouse/issues/16246). Added connection pools for odbc bridge. [#21972](https://github.com/ClickHouse/ClickHouse/pull/21972) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Speeded up reading subset of columns from File-like table engine with internal file written in column oriented data formats (Parquet, Arrow and ORC) This closes [#20129](https://github.com/ClickHouse/ClickHouse/issues/20129) Done by @keen-wolf. [#22299](https://github.com/ClickHouse/ClickHouse/pull/22299) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Correctly check structure of async distributed blocks. [#22325](https://github.com/ClickHouse/ClickHouse/pull/22325) ([Azat Khuzhin](https://github.com/azat)). +* Make `round` function to behave consistently on non-x86_64 platforms. Rounding half to nearest even (Banker's rounding) is used. [#22582](https://github.com/ClickHouse/ClickHouse/pull/22582) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Clear the rest of the screen and show cursor in `clickhouse-client` if previous program has left garbage in terminal. This closes [#16518](https://github.com/ClickHouse/ClickHouse/issues/16518). [#22634](https://github.com/ClickHouse/ClickHouse/pull/22634) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to use CTE in VIEW definition. This closes [#22491](https://github.com/ClickHouse/ClickHouse/issues/22491). [#22657](https://github.com/ClickHouse/ClickHouse/pull/22657) ([Amos Bird](https://github.com/amosbird)). +* Add metric to track how much time is spend during waiting for Buffer layer lock. [#22725](https://github.com/ClickHouse/ClickHouse/pull/22725) ([Azat Khuzhin](https://github.com/azat)). +* Allow RBAC row policy via postgresql protocol. Closes [#22658](https://github.com/ClickHouse/ClickHouse/issues/22658). PostgreSQL protocol is enabled in configuration by default. [#22755](https://github.com/ClickHouse/ClickHouse/pull/22755) ([Kseniia Sumarokova](https://github.com/kssenii)). +* MaterializeMySQL (experimental feature). Make Clickhouse to be able to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. ... [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian Frøystad](https://github.com/cfroystad)). +* `dateDiff` now works with `DateTime64` arguments (even for values outside of `DateTime` range) ... [#22931](https://github.com/ClickHouse/ClickHouse/pull/22931) ([Vasily Nemkov](https://github.com/Enmk)). +* Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix inactive_parts_to_throw_insert=0 with inactive_parts_to_delay_insert>0. [#22947](https://github.com/ClickHouse/ClickHouse/pull/22947) ([Azat Khuzhin](https://github.com/azat)). +* Respect max_part_removal_threads for ReplicatedMergeTree. [#22971](https://github.com/ClickHouse/ClickHouse/pull/22971) ([Azat Khuzhin](https://github.com/azat)). +* Fix an error handling in Poco HTTP Client for AWS. [#22973](https://github.com/ClickHouse/ClickHouse/pull/22973) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). +* When selecting from MergeTree table with NULL in WHERE condition, in rare cases, exception was thrown. This closes [#20019](https://github.com/ClickHouse/ClickHouse/issues/20019). [#22978](https://github.com/ClickHouse/ClickHouse/pull/22978) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add ability to flush buffer only in background for StorageBuffer. [#22986](https://github.com/ClickHouse/ClickHouse/pull/22986) ([Azat Khuzhin](https://github.com/azat)). +* Add ability to run clickhouse-keeper with SSL. Config settings `keeper_server.tcp_port_secure` can be used for secure interaction between client and keeper-server. `keeper_server.raft_configuration.secure` can be used to enable internal secure communication between nodes. [#22992](https://github.com/ClickHouse/ClickHouse/pull/22992) ([alesapin](https://github.com/alesapin)). +* Increase `max_uri_size` (the maximum size of URL in HTTP interface) to 1 MiB by default. This closes [#21197](https://github.com/ClickHouse/ClickHouse/issues/21197). [#22997](https://github.com/ClickHouse/ClickHouse/pull/22997) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not perform optimize_skip_unused_shards for cluster with one node. [#22999](https://github.com/ClickHouse/ClickHouse/pull/22999) ([Azat Khuzhin](https://github.com/azat)). +* Raised the threshold on max number of matches in result of the function `extractAllGroupsHorizontal`. [#23036](https://github.com/ClickHouse/ClickHouse/pull/23036) ([Vasily Nemkov](https://github.com/Enmk)). +* Implement functions `arrayHasAny`, `arrayHasAll`, `has`, `indexOf`, `countEqual` for generic case when types of array elements are different. In previous versions the functions `arrayHasAny`, `arrayHasAll` returned false and `has`, `indexOf`, `countEqual` thrown exception. Also add support for `Decimal` and big integer types in functions `has` and similar. This closes [#20272](https://github.com/ClickHouse/ClickHouse/issues/20272). [#23044](https://github.com/ClickHouse/ClickHouse/pull/23044) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix memory tracking with min_bytes_to_use_mmap_io. [#23211](https://github.com/ClickHouse/ClickHouse/pull/23211) ([Azat Khuzhin](https://github.com/azat)). +* Make function `unhex` case insensitive for compatibility with MySQL. [#23229](https://github.com/ClickHouse/ClickHouse/pull/23229) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Fix very rare bug when quorum insert with `quorum_parallel=1` is not really "quorum" because of deduplication. [#18215](https://github.com/ClickHouse/ClickHouse/pull/18215) ([filimonov](https://github.com/filimonov)). +* * Fix "unknown column" error for tables with `Merge` engine in queris with `JOIN` and aggregation. Closes [#18368](https://github.com/ClickHouse/ClickHouse/issues/18368), close [#22226](https://github.com/ClickHouse/ClickHouse/issues/22226). [#21370](https://github.com/ClickHouse/ClickHouse/pull/21370) ([Vladimir C](https://github.com/vdimir)). +* Check if table function view is used as a column. This complements https://github.com/ClickHouse/ClickHouse/pull/20350. [#21465](https://github.com/ClickHouse/ClickHouse/pull/21465) ([Amos Bird](https://github.com/amosbird)). +* Fix bug, which leads to underaggregation of data in case of enabled `optimize_aggregation_in_order` and many parts in table. Slightly improve performance of aggregation with enabled `optimize_aggregation_in_order`. [#21889](https://github.com/ClickHouse/ClickHouse/pull/21889) ([Anton Popov](https://github.com/CurtizJ)). +* Do not limit HTTP chunk size. Fixes [#21907](https://github.com/ClickHouse/ClickHouse/issues/21907). [#22322](https://github.com/ClickHouse/ClickHouse/pull/22322) ([Ivan](https://github.com/abyss7)). +* Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ClickHouseDictionarySource configuration loop. Closes [#14314](https://github.com/ClickHouse/ClickHouse/issues/14314). [#22479](https://github.com/ClickHouse/ClickHouse/pull/22479) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix bug in partial merge join with `LowCardinality`. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir C](https://github.com/vdimir)). +* Follow-up fix for [#21936](https://github.com/ClickHouse/ClickHouse/issues/21936). Also fixes [#22433](https://github.com/ClickHouse/ClickHouse/issues/22433). [#22518](https://github.com/ClickHouse/ClickHouse/pull/22518) ([Ivan](https://github.com/abyss7)). +* Fix deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UB in *Log engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)). +* Fix usage of function `map` in distributed queries. [#22588](https://github.com/ClickHouse/ClickHouse/pull/22588) ([foolchi](https://github.com/foolchi)). +* Try flush write buffer only if it is initialized. Fixes segfault when client closes connection very early [#22579](https://github.com/ClickHouse/ClickHouse/issues/22579). [#22591](https://github.com/ClickHouse/ClickHouse/pull/22591) ([nvartolomei](https://github.com/nvartolomei)). +* Fixed a bug with unlimited wait for auxiliary AWS requests. [#22594](https://github.com/ClickHouse/ClickHouse/pull/22594) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix LOGICAL_ERROR for Log with nested types w/o columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)). +* Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). +* Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix approx total rows accounting for reverse reading from MergeTree. [#22726](https://github.com/ClickHouse/ClickHouse/pull/22726) ([Azat Khuzhin](https://github.com/azat)). +* * Revert "Move conditions from JOIN ON to WHERE" (ClickHouse/ClickHouse[#19685](https://github.com/ClickHouse/ClickHouse/issues/19685)), close [#22399](https://github.com/ClickHouse/ClickHouse/issues/22399), close [#21671](https://github.com/ClickHouse/ClickHouse/issues/21671). [#22753](https://github.com/ClickHouse/ClickHouse/pull/22753) ([Vladimir C](https://github.com/vdimir)). +* Fix pushdown of `HAVING` in case, when filter column is used in aggregation. [#22763](https://github.com/ClickHouse/ClickHouse/pull/22763) ([Anton Popov](https://github.com/CurtizJ)). +* LIVE VIEW (experimental feature). Fix possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, see https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e. [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix bytes_allocated for sparse_hashed dictionaries. [#22867](https://github.com/ClickHouse/ClickHouse/pull/22867) ([Azat Khuzhin](https://github.com/azat)). +* Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fixed `formatDateTime()` on `DateTime64` and "%C" format specifier fixed `toDateTime64()` for large values and non-zero scale. ... [#22937](https://github.com/ClickHouse/ClickHouse/pull/22937) ([Vasily Nemkov](https://github.com/Enmk)). +* Fix usage of constant columns of type `Map` with nullable values. [#22939](https://github.com/ClickHouse/ClickHouse/pull/22939) ([Anton Popov](https://github.com/CurtizJ)). +* Simplify debian packages. This fixes [#21698](https://github.com/ClickHouse/ClickHouse/issues/21698). [#22976](https://github.com/ClickHouse/ClickHouse/pull/22976) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error `Cannot find column in ActionsDAG result` which may happen if subquery uses `untuple`. Fixes [#22290](https://github.com/ClickHouse/ClickHouse/issues/22290). [#22991](https://github.com/ClickHouse/ClickHouse/pull/22991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed `Table .inner_id... doesn't exist` error when selecting from Materialized View after detaching it from Atomic database and attaching back. [#23047](https://github.com/ClickHouse/ClickHouse/pull/23047) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Server might fail to start if `data_type_default_nullable` setting is enabled in default profile, it's fixed. Fixes [#22573](https://github.com/ClickHouse/ClickHouse/issues/22573). [#23185](https://github.com/ClickHouse/ClickHouse/pull/23185) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix missing whitespace in some exception messages about `LowCardinality` type. [#23207](https://github.com/ClickHouse/ClickHouse/pull/23207) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed missing semicolon in exception message. The user may find this exception message unpleasant to read. [#23208](https://github.com/ClickHouse/ClickHouse/pull/23208) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix reading from ODBC when there are many long column names in a table. Closes [#8853](https://github.com/ClickHouse/ClickHouse/issues/8853). [#23215](https://github.com/ClickHouse/ClickHouse/pull/23215) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Build/Testing/Packaging Improvement +* Add on-demand check for clickhouse Keeper. [#22373](https://github.com/ClickHouse/ClickHouse/pull/22373) ([alesapin](https://github.com/alesapin)). +* Disable incompatible libraries (platform specific typically) on ppc64le ... [#22475](https://github.com/ClickHouse/ClickHouse/pull/22475) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Allow building with unbundled xz (lzma) using USE_INTERNAL_XZ_LIBRARY=OFF ... [#22571](https://github.com/ClickHouse/ClickHouse/pull/22571) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Allow query profiling only on x86_64. See [#15174](https://github.com/ClickHouse/ClickHouse/issues/15174)#issuecomment-812954965 and [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638)#issuecomment-703805337. This closes [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638). [#22580](https://github.com/ClickHouse/ClickHouse/pull/22580) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adjust some tests to output identical results on amd64 and aarch64 (qemu). The result was depending on implementation specific CPU behaviour. [#22590](https://github.com/ClickHouse/ClickHouse/pull/22590) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix some tests on AArch64 platform. [#22596](https://github.com/ClickHouse/ClickHouse/pull/22596) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ClickHouse Keeper build for MacOS. [#22860](https://github.com/ClickHouse/ClickHouse/pull/22860) ([alesapin](https://github.com/alesapin)). + +#### Other +* Fix some points from this comment https://github.com/ClickHouse/ClickHouse/pull/19516#issuecomment-782047840. [#22323](https://github.com/ClickHouse/ClickHouse/pull/22323) ([Ivan](https://github.com/abyss7)). + +#### Build/Packaging/Testing Improvement + +* Build `jemalloc` with support for [heap profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling). [#22834](https://github.com/ClickHouse/ClickHouse/pull/22834) ([nvartolomei](https://github.com/nvartolomei)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Error message reads better'. [#22983](https://github.com/ClickHouse/ClickHouse/pull/22983) ([Igor O'sten](https://github.com/borodark)). + diff --git a/docs/changelogs/v21.5.2.25-prestable.md b/docs/changelogs/v21.5.2.25-prestable.md new file mode 100644 index 00000000000..45e784218da --- /dev/null +++ b/docs/changelogs/v21.5.2.25-prestable.md @@ -0,0 +1,40 @@ +### ClickHouse release v21.5.2.25-prestable FIXME as compared to v21.5.1.6601-prestable + +#### Improvement +* Backported in [#23342](https://github.com/ClickHouse/ClickHouse/issues/23342): Disable settings `use_hedged_requests` and `async_socket_for_remote` because there is an evidence that it may cause issues. [#23261](https://github.com/ClickHouse/ClickHouse/pull/23261) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23678](https://github.com/ClickHouse/ClickHouse/issues/23678): Fixed `quantile(s)TDigest`. Added special handling of singleton centroids according to tdunning/t-digest 3.2+. Also a bug with over-compression of centroids in implementation of earlier version of the algorithm was fixed. [#23314](https://github.com/ClickHouse/ClickHouse/pull/23314) ([Vladimir Chebotarev](https://github.com/excitoon)). + +#### Bug Fix +* Backported in [#23343](https://github.com/ClickHouse/ClickHouse/issues/23343): This PR fixes a crash on shutdown which happened because of currentConnections() could return zero while some connections were still alive. [#23154](https://github.com/ClickHouse/ClickHouse/pull/23154) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#23405](https://github.com/ClickHouse/ClickHouse/issues/23405): QueryAliasVisitor to prefer alias for ASTWithAlias if subquery was optimized to constant. Fixes [#22924](https://github.com/ClickHouse/ClickHouse/issues/22924). Fixes [#10401](https://github.com/ClickHouse/ClickHouse/issues/10401). [#23191](https://github.com/ClickHouse/ClickHouse/pull/23191) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23383](https://github.com/ClickHouse/ClickHouse/issues/23383): Fixed `Not found column` error when selecting from `MaterializeMySQL` with condition on key column. Fixes [#22432](https://github.com/ClickHouse/ClickHouse/issues/22432). [#23200](https://github.com/ClickHouse/ClickHouse/pull/23200) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23386](https://github.com/ClickHouse/ClickHouse/issues/23386): Fixed the behavior when disabling `input_format_with_names_use_header ` setting discards all the input with CSVWithNames format. This fixes [#22406](https://github.com/ClickHouse/ClickHouse/issues/22406). [#23202](https://github.com/ClickHouse/ClickHouse/pull/23202) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#23340](https://github.com/ClickHouse/ClickHouse/issues/23340): Add type conversion for optimize_skip_unused_shards_rewrite_in (fixes `use-of-uninitialized-value` with `optimize_skip_unused_shards_rewrite_in`). [#23219](https://github.com/ClickHouse/ClickHouse/pull/23219) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#23341](https://github.com/ClickHouse/ClickHouse/issues/23341): Fixed simple key dictionary from DDL creation if primary key is not first attribute. Fixes [#23236](https://github.com/ClickHouse/ClickHouse/issues/23236). [#23262](https://github.com/ClickHouse/ClickHouse/pull/23262) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23449](https://github.com/ClickHouse/ClickHouse/issues/23449): Fixed very rare (distributed) race condition between creation and removal of ReplicatedMergeTree tables. It might cause exceptions like `node doesn't exist` on attempt to create replicated table. Fixes [#21419](https://github.com/ClickHouse/ClickHouse/issues/21419). [#23294](https://github.com/ClickHouse/ClickHouse/pull/23294) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23578](https://github.com/ClickHouse/ClickHouse/issues/23578): Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23427](https://github.com/ClickHouse/ClickHouse/issues/23427): Fix possible crash in case if `unknown packet` was received form remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#23675](https://github.com/ClickHouse/ClickHouse/issues/23675): Don't relax NOT conditions during partition pruning. This fixes [#23305](https://github.com/ClickHouse/ClickHouse/issues/23305) and [#21539](https://github.com/ClickHouse/ClickHouse/issues/21539). [#23310](https://github.com/ClickHouse/ClickHouse/pull/23310) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#23581](https://github.com/ClickHouse/ClickHouse/issues/23581): * Fix bug in dict join with join_algorithm = 'auto'. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#23428](https://github.com/ClickHouse/ClickHouse/issues/23428): Fix possible `Block structure mismatch` error for queries with `UNION` which could possibly happen after filter-push-down optimization. Fixes [#23029](https://github.com/ClickHouse/ClickHouse/issues/23029). [#23359](https://github.com/ClickHouse/ClickHouse/pull/23359) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#23642](https://github.com/ClickHouse/ClickHouse/issues/23642): Fix incompatible constant expression generation during partition pruning based on virtual columns. This fixes https://github.com/ClickHouse/ClickHouse/pull/21401#discussion_r611888913. [#23366](https://github.com/ClickHouse/ClickHouse/pull/23366) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#23453](https://github.com/ClickHouse/ClickHouse/issues/23453): `ORDER BY` with `COLLATE` was not working correctly if the column is in primary key (or is a monotonic function of it) and the setting `optimize_read_in_order` is not turned off. This closes [#22379](https://github.com/ClickHouse/ClickHouse/issues/22379). Workaround for older versions: turn the setting `optimize_read_in_order` off. [#23375](https://github.com/ClickHouse/ClickHouse/pull/23375) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23510](https://github.com/ClickHouse/ClickHouse/issues/23510): Remove support for `argMin` and `argMax` for single `Tuple` argument. The code was not memory-safe. The feature was added by mistake and it is confusing for people. These functions can be reintroduced under different names later. This fixes [#22384](https://github.com/ClickHouse/ClickHouse/issues/22384) and reverts [#17359](https://github.com/ClickHouse/ClickHouse/issues/17359). [#23393](https://github.com/ClickHouse/ClickHouse/pull/23393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23584](https://github.com/ClickHouse/ClickHouse/issues/23584): Allow to move more conditions to `PREWHERE` as it was before version 21.1. Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#23477](https://github.com/ClickHouse/ClickHouse/issues/23477): Kafka storage may support `arrow` and `arrowstream` format messages. [#23415](https://github.com/ClickHouse/ClickHouse/pull/23415) ([Chao Ma](https://github.com/godliness)). +* Backported in [#23587](https://github.com/ClickHouse/ClickHouse/issues/23587): Fixed `Cannot unlink file` error on unsuccessful creation of ReplicatedMergeTree table with multidisk configuration. This closes [#21755](https://github.com/ClickHouse/ClickHouse/issues/21755). [#23433](https://github.com/ClickHouse/ClickHouse/pull/23433) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23497](https://github.com/ClickHouse/ClickHouse/issues/23497): - Bug fix for `deltaSum` aggregate function in counter reset case ... [#23437](https://github.com/ClickHouse/ClickHouse/pull/23437) ([Russ Frank](https://github.com/rf)). +* Backported in [#23491](https://github.com/ClickHouse/ClickHouse/issues/23491): Fix bug that does not allow cast from empty array literal, to array with dimensions greater than 1. Closes [#14476](https://github.com/ClickHouse/ClickHouse/issues/14476). [#23456](https://github.com/ClickHouse/ClickHouse/pull/23456) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23589](https://github.com/ClickHouse/ClickHouse/issues/23589): Fix corner cases in vertical merges with `ReplacingMergeTree`. In rare cases they could lead to fails of merges with exceptions like `Incomplete granules are not allowed while blocks are granules size`. [#23459](https://github.com/ClickHouse/ClickHouse/pull/23459) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#23535](https://github.com/ClickHouse/ClickHouse/issues/23535): When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#23532](https://github.com/ClickHouse/ClickHouse/issues/23532): Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23613](https://github.com/ClickHouse/ClickHouse/issues/23613): Fix restart / stop command hanging. Closes [#20214](https://github.com/ClickHouse/ClickHouse/issues/20214). [#23552](https://github.com/ClickHouse/ClickHouse/pull/23552) ([filimonov](https://github.com/filimonov)). +* Backported in [#23627](https://github.com/ClickHouse/ClickHouse/issues/23627): Fix misinterpretation of some `LIKE` expressions with escape sequences. [#23610](https://github.com/ClickHouse/ClickHouse/pull/23610) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#23692](https://github.com/ClickHouse/ClickHouse/issues/23692): Fixed server fault when inserting data through HTTP caused an exception. This fixes [#23512](https://github.com/ClickHouse/ClickHouse/issues/23512). [#23643](https://github.com/ClickHouse/ClickHouse/pull/23643) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#23694](https://github.com/ClickHouse/ClickHouse/issues/23694): Added an exception in case of completely the same values in both samples in aggregate function `mannWhitneyUTest`. This fixes [#23646](https://github.com/ClickHouse/ClickHouse/issues/23646). [#23654](https://github.com/ClickHouse/ClickHouse/pull/23654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#23757](https://github.com/ClickHouse/ClickHouse/issues/23757): Avoid possible "Cannot schedule a task" error (in case some exception had been occurred) on INSERT into Distributed. [#23744](https://github.com/ClickHouse/ClickHouse/pull/23744) ([Azat Khuzhin](https://github.com/azat)). + +#### NO CL CATEGORY + +* Backported in [#23347](https://github.com/ClickHouse/ClickHouse/issues/23347):. [#23334](https://github.com/ClickHouse/ClickHouse/pull/23334) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.5.3.1-prestable.md b/docs/changelogs/v21.5.3.1-prestable.md new file mode 100644 index 00000000000..24b29d1495e --- /dev/null +++ b/docs/changelogs/v21.5.3.1-prestable.md @@ -0,0 +1,9 @@ +### ClickHouse release v21.5.3.1-prestable FIXME as compared to v21.5.2.25-prestable + +#### Bug Fix +* Backported in [#23818](https://github.com/ClickHouse/ClickHouse/issues/23818): Fix crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#23798](https://github.com/ClickHouse/ClickHouse/issues/23798): Fixed remote JDBC bridge timeout connection issue. Closes [#9609](https://github.com/ClickHouse/ClickHouse/issues/9609). [#23771](https://github.com/ClickHouse/ClickHouse/pull/23771) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#23815](https://github.com/ClickHouse/ClickHouse/issues/23815): Fix `CLEAR COLUMN` does not work when it is referenced by materialized view. Close [#23764](https://github.com/ClickHouse/ClickHouse/issues/23764). [#23781](https://github.com/ClickHouse/ClickHouse/pull/23781) ([flynn](https://github.com/ucasfl)). +* Backported in [#23832](https://github.com/ClickHouse/ClickHouse/issues/23832): Fix error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` and `use_hedged_requests`. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#23927](https://github.com/ClickHouse/ClickHouse/issues/23927): HashedDictionary complex key update field initial load fix. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.4.6-prestable.md b/docs/changelogs/v21.5.4.6-prestable.md new file mode 100644 index 00000000000..c9e040309cc --- /dev/null +++ b/docs/changelogs/v21.5.4.6-prestable.md @@ -0,0 +1,8 @@ +### ClickHouse release v21.5.4.6-prestable FIXME as compared to v21.5.3.1-prestable + +#### Bug Fix +* Backported in [#23972](https://github.com/ClickHouse/ClickHouse/issues/23972): Fixed a bug in recovery of staled `ReplicatedMergeTree` replica. Some metadata updates could be ignored by staled replica if `ALTER` query was executed during downtime of the replica. [#23742](https://github.com/ClickHouse/ClickHouse/pull/23742) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#23956](https://github.com/ClickHouse/ClickHouse/issues/23956): Fix keys metrics accounting for CACHE() dictionary with duplicates in the source (leads to `DictCacheKeysRequestedMiss` overflows). [#23929](https://github.com/ClickHouse/ClickHouse/pull/23929) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#24001](https://github.com/ClickHouse/ClickHouse/issues/24001): Fix SIGSEGV for external GROUP BY and overflow row (i.e. queries like `SELECT FROM GROUP BY WITH TOTALS SETTINGS max_bytes_before_external_group_by>0, max_rows_to_group_by>0, group_by_overflow_mode='any', totals_mode='before_having'`). [#23962](https://github.com/ClickHouse/ClickHouse/pull/23962) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#24033](https://github.com/ClickHouse/ClickHouse/issues/24033): Fix crash in MergeJoin, close [#24010](https://github.com/ClickHouse/ClickHouse/issues/24010). [#24013](https://github.com/ClickHouse/ClickHouse/pull/24013) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.5.5.12-stable.md b/docs/changelogs/v21.5.5.12-stable.md new file mode 100644 index 00000000000..b49e2c60b08 --- /dev/null +++ b/docs/changelogs/v21.5.5.12-stable.md @@ -0,0 +1,11 @@ +### ClickHouse release v21.5.5.12-stable FIXME as compared to v21.5.4.6-prestable + +#### Bug Fix +* Backported in [#24156](https://github.com/ClickHouse/ClickHouse/issues/24156): Kafka storage may support parquet format messages. [#23412](https://github.com/ClickHouse/ClickHouse/pull/23412) ([Chao Ma](https://github.com/godliness)). +* Backported in [#24187](https://github.com/ClickHouse/ClickHouse/issues/24187): Some `ALTER PARTITION` queries might cause `Part A intersects previous part B` and `Unexpected merged part C intersecting drop range D` errors in replication queue. It's fixed. Fixes [#23296](https://github.com/ClickHouse/ClickHouse/issues/23296). [#23997](https://github.com/ClickHouse/ClickHouse/pull/23997) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24304](https://github.com/ClickHouse/ClickHouse/issues/24304): Fixed using const `DateTime` value vs `DateTime64` column in WHERE. ... [#24100](https://github.com/ClickHouse/ClickHouse/pull/24100) ([Vasily Nemkov](https://github.com/Enmk)). +* Backported in [#24142](https://github.com/ClickHouse/ClickHouse/issues/24142): Bug: explain pipeline with` select xxx final `shows wrong pipeline: ``` dell123 :) explain pipeline select z from prewhere_move_select_final final;. [#24116](https://github.com/ClickHouse/ClickHouse/pull/24116) ([hexiaoting](https://github.com/hexiaoting)). +* Backported in [#24214](https://github.com/ClickHouse/ClickHouse/issues/24214): Fix race condition which could happen in RBAC under a heavy load. This PR fixes [#24090](https://github.com/ClickHouse/ClickHouse/issues/24090), [#24134](https://github.com/ClickHouse/ClickHouse/issues/24134),. [#24176](https://github.com/ClickHouse/ClickHouse/pull/24176) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#24242](https://github.com/ClickHouse/ClickHouse/issues/24242): Fix abnormal server termination due to hdfs becoming not accessible during query execution. Closes [#24117](https://github.com/ClickHouse/ClickHouse/issues/24117). [#24191](https://github.com/ClickHouse/ClickHouse/pull/24191) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#24239](https://github.com/ClickHouse/ClickHouse/issues/24239): Fix wrong typo at StorageMemory, this bug was introduced at [#15127](https://github.com/ClickHouse/ClickHouse/issues/15127), now fixed, Closes [#24192](https://github.com/ClickHouse/ClickHouse/issues/24192). [#24193](https://github.com/ClickHouse/ClickHouse/pull/24193) ([张中南](https://github.com/plugine)). + diff --git a/docs/changelogs/v21.5.6.6-stable.md b/docs/changelogs/v21.5.6.6-stable.md new file mode 100644 index 00000000000..e6160dfa784 --- /dev/null +++ b/docs/changelogs/v21.5.6.6-stable.md @@ -0,0 +1,17 @@ +### ClickHouse release v21.5.6.6-stable FIXME as compared to v21.5.5.12-stable + +#### Bug Fix +* Backported in [#24170](https://github.com/ClickHouse/ClickHouse/issues/24170): Fix a rare bug that could lead to a partially initialized table that can serve write requests (insert/alter/so on). Now such tables will be in readonly mode. [#24122](https://github.com/ClickHouse/ClickHouse/pull/24122) ([alesapin](https://github.com/alesapin)). +* Backported in [#24332](https://github.com/ClickHouse/ClickHouse/issues/24332): Set `max_threads = 1` to fix mutation fail of StorageMemory. Closes [#24274](https://github.com/ClickHouse/ClickHouse/issues/24274). [#24275](https://github.com/ClickHouse/ClickHouse/pull/24275) ([flynn](https://github.com/ucasfl)). +* Backported in [#24382](https://github.com/ClickHouse/ClickHouse/issues/24382): Allow empty HTTP headers. Fixes [#23901](https://github.com/ClickHouse/ClickHouse/issues/23901). [#24285](https://github.com/ClickHouse/ClickHouse/pull/24285) ([Ivan](https://github.com/abyss7)). +* Backported in [#24351](https://github.com/ClickHouse/ClickHouse/issues/24351): Fixed a bug in moving Materialized View from Ordinary to Atomic database (`RENAME TABLE` query). Now inner table is moved to new database together with Materialized View. Fixes [#23926](https://github.com/ClickHouse/ClickHouse/issues/23926). [#24309](https://github.com/ClickHouse/ClickHouse/pull/24309) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24509](https://github.com/ClickHouse/ClickHouse/issues/24509): Fix drop partition with intersect fake parts. In rare cases there might be parts with mutation version greater than current block number. [#24321](https://github.com/ClickHouse/ClickHouse/pull/24321) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#24540](https://github.com/ClickHouse/ClickHouse/issues/24540): Fix incorrect monotonicity of toWeek function. This fixes [#24422](https://github.com/ClickHouse/ClickHouse/issues/24422) . This bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/5212 , and was exposed later by smarter partition pruner. [#24446](https://github.com/ClickHouse/ClickHouse/pull/24446) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#24557](https://github.com/ClickHouse/ClickHouse/issues/24557): Fixed the behavior when query `SYSTEM RESTART REPLICA` or `SYSTEM SYNC REPLICA` is being processed infinitely. This was detected on server with extremely little amount of RAM. [#24457](https://github.com/ClickHouse/ClickHouse/pull/24457) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#24598](https://github.com/ClickHouse/ClickHouse/issues/24598): Fix usage of tuples in `CREATE .. AS SELECT` queries. [#24464](https://github.com/ClickHouse/ClickHouse/pull/24464) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#24601](https://github.com/ClickHouse/ClickHouse/issues/24601): Enable reading of subcolumns for distributed tables. [#24472](https://github.com/ClickHouse/ClickHouse/pull/24472) ([Anton Popov](https://github.com/CurtizJ)). + +#### NO CL CATEGORY + +* Backported in [#24750](https://github.com/ClickHouse/ClickHouse/issues/24750):. [#23276](https://github.com/ClickHouse/ClickHouse/pull/23276) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.5.7.9-stable.md b/docs/changelogs/v21.5.7.9-stable.md new file mode 100644 index 00000000000..9586e398547 --- /dev/null +++ b/docs/changelogs/v21.5.7.9-stable.md @@ -0,0 +1,42 @@ +### ClickHouse release v21.5.7.9-stable FIXME as compared to v21.5.6.6-stable + +#### Improvement +* Backported in [#24581](https://github.com/ClickHouse/ClickHouse/issues/24581): Disable min_bytes_to_use_mmap_io by default. [#23322](https://github.com/ClickHouse/ClickHouse/pull/23322) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#25222](https://github.com/ClickHouse/ClickHouse/issues/25222): Here will be listed all the bugs that I am gonna to fix in this PR. [#23518](https://github.com/ClickHouse/ClickHouse/pull/23518) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#24793](https://github.com/ClickHouse/ClickHouse/issues/24793): Avoid hiding errors like `Limit for rows or bytes to read exceeded` for scalar subqueries. [#24545](https://github.com/ClickHouse/ClickHouse/pull/24545) ([nvartolomei](https://github.com/nvartolomei)). + +#### Bug Fix +* Backported in [#24928](https://github.com/ClickHouse/ClickHouse/issues/24928): Fix implementation of connection pool of PostgreSQL engine. Closes [#23897](https://github.com/ClickHouse/ClickHouse/issues/23897). [#23909](https://github.com/ClickHouse/ClickHouse/pull/23909) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#24568](https://github.com/ClickHouse/ClickHouse/issues/24568): Use old modulo function version when used in partition key. Closes [#23508](https://github.com/ClickHouse/ClickHouse/issues/23508). [#24157](https://github.com/ClickHouse/ClickHouse/pull/24157) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#24726](https://github.com/ClickHouse/ClickHouse/issues/24726): In "multipart/form-data" message consider the CRLF preceding a boundary as part of it. Fixes [#23905](https://github.com/ClickHouse/ClickHouse/issues/23905). [#24399](https://github.com/ClickHouse/ClickHouse/pull/24399) ([Ivan](https://github.com/abyss7)). +* Backported in [#24824](https://github.com/ClickHouse/ClickHouse/issues/24824): - Fixed the deadlock that can happen during LDAP role (re)mapping, when LDAP group is mapped to a nonexistent local role. [#24431](https://github.com/ClickHouse/ClickHouse/pull/24431) ([Denis Glazachev](https://github.com/traceon)). +* Backported in [#24772](https://github.com/ClickHouse/ClickHouse/issues/24772): Fix bug which can lead to ZooKeeper client hung inside clickhouse-server. [#24721](https://github.com/ClickHouse/ClickHouse/pull/24721) ([alesapin](https://github.com/alesapin)). +* Backported in [#24935](https://github.com/ClickHouse/ClickHouse/issues/24935): - If ZooKeeper connection was lost and replica was cloned after restoring the connection, its replication queue might contain outdated entries. It's fixed. - Fixed crash when replication queue contains intersecting virtual parts. It may rarely happen if some data part was lost. Print error in log instead of terminating. [#24777](https://github.com/ClickHouse/ClickHouse/pull/24777) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24854](https://github.com/ClickHouse/ClickHouse/issues/24854): Fix bug when exception `Mutation was killed` can be thrown to the client on mutation wait when mutation not loaded into memory yet. [#24809](https://github.com/ClickHouse/ClickHouse/pull/24809) ([alesapin](https://github.com/alesapin)). +* Backported in [#24986](https://github.com/ClickHouse/ClickHouse/issues/24986): Allow NULL values in postgresql protocol. Closes [#22622](https://github.com/ClickHouse/ClickHouse/issues/22622). [#24857](https://github.com/ClickHouse/ClickHouse/pull/24857) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#25481](https://github.com/ClickHouse/ClickHouse/issues/25481): Fix "Missing columns" exception when joining Distributed Materialized View. [#24870](https://github.com/ClickHouse/ClickHouse/pull/24870) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#24915](https://github.com/ClickHouse/ClickHouse/issues/24915): Fix extremely rare bug on low-memory servers which can lead to the inability to perform merges without restart. Possibly fixes [#24603](https://github.com/ClickHouse/ClickHouse/issues/24603). [#24872](https://github.com/ClickHouse/ClickHouse/pull/24872) ([alesapin](https://github.com/alesapin)). +* Backported in [#25183](https://github.com/ClickHouse/ClickHouse/issues/25183): Fixed bug with declaring S3 disk at root of bucket. Earlier, it reported an error: ``` [heather] 2021.05.10 02:11:11.932234 [ 72790 ] {2ff80b7b-ec53-41cb-ac35-19bb390e1759} executeQuery: Code: 36, e.displayText() = DB::Exception: Key name is empty in path style S3 URI: (http://172.17.0.2/bucket/) (version 21.6.1.1) (from 127.0.0.1:47994) (in query: SELECT policy_name FROM system.storage_policies), Stack trace (when copying this message, always include the lines below):. [#24898](https://github.com/ClickHouse/ClickHouse/pull/24898) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#24950](https://github.com/ClickHouse/ClickHouse/issues/24950): Fix possible heap-buffer-overflow in Arrow. [#24922](https://github.com/ClickHouse/ClickHouse/pull/24922) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#25024](https://github.com/ClickHouse/ClickHouse/issues/25024): Fix limit/offset settings for distributed queries (ignore on the remote nodes). [#24940](https://github.com/ClickHouse/ClickHouse/pull/24940) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#25048](https://github.com/ClickHouse/ClickHouse/issues/25048): Fix extremely rare error `Tagging already tagged part` in replication queue during concurrent `alter move/replace partition`. Possibly fixes [#22142](https://github.com/ClickHouse/ClickHouse/issues/22142). [#24961](https://github.com/ClickHouse/ClickHouse/pull/24961) ([alesapin](https://github.com/alesapin)). +* Backported in [#25365](https://github.com/ClickHouse/ClickHouse/issues/25365): Fix serialization of splitted nested messages in Protobuf format. This PR fixes [#24647](https://github.com/ClickHouse/ClickHouse/issues/24647). [#25000](https://github.com/ClickHouse/ClickHouse/pull/25000) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#25101](https://github.com/ClickHouse/ClickHouse/issues/25101): Distinguish KILL MUTATION for different tables (fixes unexpected `Cancelled mutating parts` error). [#25025](https://github.com/ClickHouse/ClickHouse/pull/25025) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#25558](https://github.com/ClickHouse/ClickHouse/issues/25558): Fix bug which allows creating tables with columns referencing themselves like `a UInt32 ALIAS a + 1` or `b UInt32 MATERIALIZED b`. Fixes [#24910](https://github.com/ClickHouse/ClickHouse/issues/24910), [#24292](https://github.com/ClickHouse/ClickHouse/issues/24292). [#25059](https://github.com/ClickHouse/ClickHouse/pull/25059) ([alesapin](https://github.com/alesapin)). +* Backported in [#25104](https://github.com/ClickHouse/ClickHouse/issues/25104): Fix bug with constant maps in mapContains that lead to error `empty column was returned by function mapContains`. Closes [#25077](https://github.com/ClickHouse/ClickHouse/issues/25077). [#25080](https://github.com/ClickHouse/ClickHouse/pull/25080) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#25141](https://github.com/ClickHouse/ClickHouse/issues/25141): Fix crash in query with cross join and `joined_subquery_requires_alias = 0`. Fixes [#24011](https://github.com/ClickHouse/ClickHouse/issues/24011). [#25082](https://github.com/ClickHouse/ClickHouse/pull/25082) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25214](https://github.com/ClickHouse/ClickHouse/issues/25214): Fixed an error which occurred while inserting a subset of columns using CSVWithNames format. Fixes [#25129](https://github.com/ClickHouse/ClickHouse/issues/25129). [#25169](https://github.com/ClickHouse/ClickHouse/pull/25169) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#25353](https://github.com/ClickHouse/ClickHouse/issues/25353): Fix TOCTOU error in installation script. [#25277](https://github.com/ClickHouse/ClickHouse/pull/25277) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#25471](https://github.com/ClickHouse/ClickHouse/issues/25471): Fix joinGetOrNull with not-nullable columns. This fixes [#24261](https://github.com/ClickHouse/ClickHouse/issues/24261). [#25288](https://github.com/ClickHouse/ClickHouse/pull/25288) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#25357](https://github.com/ClickHouse/ClickHouse/issues/25357): Fix Logical Error Cannot sum Array/Tuple in min/maxMap. [#25298](https://github.com/ClickHouse/ClickHouse/pull/25298) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#25438](https://github.com/ClickHouse/ClickHouse/issues/25438): Support `SimpleAggregateFunction(LowCardinality)` for `SummingMergeTree`. Fixes [#25134](https://github.com/ClickHouse/ClickHouse/issues/25134). [#25300](https://github.com/ClickHouse/ClickHouse/pull/25300) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25362](https://github.com/ClickHouse/ClickHouse/issues/25362): On ZooKeeper connection loss `ReplicatedMergeTree` table might wait for background operations to complete before trying to reconnect. It's fixed, now background operations are stopped forcefully. [#25306](https://github.com/ClickHouse/ClickHouse/pull/25306) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25386](https://github.com/ClickHouse/ClickHouse/issues/25386): Fix the possibility of non-deterministic behaviour of the `quantileDeterministic` function and similar. This closes [#20480](https://github.com/ClickHouse/ClickHouse/issues/20480). [#25313](https://github.com/ClickHouse/ClickHouse/pull/25313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#25456](https://github.com/ClickHouse/ClickHouse/issues/25456): Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25408](https://github.com/ClickHouse/ClickHouse/issues/25408): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#25504](https://github.com/ClickHouse/ClickHouse/issues/25504): Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Backport [#24721](https://github.com/ClickHouse/ClickHouse/issues/24721) to 21.5: Remove endless `wait` from ZooKeeper client"'. [#24798](https://github.com/ClickHouse/ClickHouse/pull/24798) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.5.8.21-stable.md b/docs/changelogs/v21.5.8.21-stable.md new file mode 100644 index 00000000000..4d9ffd687eb --- /dev/null +++ b/docs/changelogs/v21.5.8.21-stable.md @@ -0,0 +1,13 @@ +### ClickHouse release v21.5.8.21-stable FIXME as compared to v21.5.7.9-stable + +#### Bug Fix +* Backported in [#25849](https://github.com/ClickHouse/ClickHouse/issues/25849): `CAST` from `Date` to `DateTime` (or `DateTime64`) was not using the timezone of the `DateTime` type. It can also affect the comparison between `Date` and `DateTime`. Inference of the common type for `Date` and `DateTime` also was not using the corresponding timezone. It affected the results of function `if` and array construction. Closes [#24128](https://github.com/ClickHouse/ClickHouse/issues/24128). [#24129](https://github.com/ClickHouse/ClickHouse/pull/24129) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#25679](https://github.com/ClickHouse/ClickHouse/issues/25679): Fixed bug in deserialization of random generator state with might cause some data types such as `AggregateFunction(groupArraySample(N), T))` to behave in a non-deterministic way. [#24538](https://github.com/ClickHouse/ClickHouse/pull/24538) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25556](https://github.com/ClickHouse/ClickHouse/issues/25556): Fix potential crash when calculating aggregate function states by aggregation of aggregate function states of other aggregate functions (not a practical use case). See [#24523](https://github.com/ClickHouse/ClickHouse/issues/24523). [#25015](https://github.com/ClickHouse/ClickHouse/pull/25015) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#25767](https://github.com/ClickHouse/ClickHouse/issues/25767): Fix assertion in PREWHERE with non-uint8 type, close [#19589](https://github.com/ClickHouse/ClickHouse/issues/19589). [#25484](https://github.com/ClickHouse/ClickHouse/pull/25484) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#25636](https://github.com/ClickHouse/ClickHouse/issues/25636): Fix wrong totals for query `WITH TOTALS` and `WITH FILL`. Fixes [#20872](https://github.com/ClickHouse/ClickHouse/issues/20872). [#25539](https://github.com/ClickHouse/ClickHouse/pull/25539) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#25653](https://github.com/ClickHouse/ClickHouse/issues/25653): Fix null pointer dereference in `EXPLAIN AST` without query. [#25631](https://github.com/ClickHouse/ClickHouse/pull/25631) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25715](https://github.com/ClickHouse/ClickHouse/issues/25715): `REPLACE PARTITION` might be ignored in rare cases if the source partition was empty. It's fixed. Fixes [#24869](https://github.com/ClickHouse/ClickHouse/issues/24869). [#25665](https://github.com/ClickHouse/ClickHouse/pull/25665) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25695](https://github.com/ClickHouse/ClickHouse/issues/25695): Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25754](https://github.com/ClickHouse/ClickHouse/issues/25754): Fix data race when querying `system.clusters` while reloading the cluster configuration at the same time. [#25737](https://github.com/ClickHouse/ClickHouse/pull/25737) ([Amos Bird](https://github.com/amosbird)). + diff --git a/docs/changelogs/v21.5.9.4-stable.md b/docs/changelogs/v21.5.9.4-stable.md new file mode 100644 index 00000000000..17ef067194b --- /dev/null +++ b/docs/changelogs/v21.5.9.4-stable.md @@ -0,0 +1,6 @@ +### ClickHouse release v21.5.9.4-stable FIXME as compared to v21.5.8.21-stable + +#### Bug Fix +* Backported in [#25958](https://github.com/ClickHouse/ClickHouse/issues/25958): Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). +* Backported in [#26144](https://github.com/ClickHouse/ClickHouse/issues/26144): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.6.1.6891-prestable.md b/docs/changelogs/v21.6.1.6891-prestable.md new file mode 100644 index 00000000000..7750cac32eb --- /dev/null +++ b/docs/changelogs/v21.6.1.6891-prestable.md @@ -0,0 +1,159 @@ +### ClickHouse release v21.6.1.6891-prestable FIXME as compared to v21.5.1.6601-prestable + +#### New Feature +* Add projection support for MergeTree* tables. [#20202](https://github.com/ClickHouse/ClickHouse/pull/20202) ([Amos Bird](https://github.com/amosbird)). +* Add back `indexHint` function. This is for [#21238](https://github.com/ClickHouse/ClickHouse/issues/21238) . This reverts https://github.com/ClickHouse/ClickHouse/pull/9542 . This fixes [#9540](https://github.com/ClickHouse/ClickHouse/issues/9540) . [#21304](https://github.com/ClickHouse/ClickHouse/pull/21304) ([Amos Bird](https://github.com/amosbird)). +* 1. Add aggregate function sumCount. This function returns a tuple of two fields: sum and count. [#21337](https://github.com/ClickHouse/ClickHouse/pull/21337) ([hexiaoting](https://github.com/hexiaoting)). +* Added less secure IMDS credentials provider for S3 which works under docker correctly. [#21852](https://github.com/ClickHouse/ClickHouse/pull/21852) ([Vladimir Chebotarev](https://github.com/excitoon)). +* - New aggregate function `deltaSumTimestamp` for summing the difference between consecutive rows while maintaining ordering during merge by storing timestamps. [#21888](https://github.com/ClickHouse/ClickHouse/pull/21888) ([Russ Frank](https://github.com/rf)). +* - LDAP: implemented user DN detection functionality to use when mapping Active Directory groups to ClickHouse roles. [#22228](https://github.com/ClickHouse/ClickHouse/pull/22228) ([Denis Glazachev](https://github.com/traceon)). +* Introduce a new function: arrayProduct which accept an array as the parameter, and return the product of all the elements in array. Close issue: [#21613](https://github.com/ClickHouse/ClickHouse/issues/21613). [#22242](https://github.com/ClickHouse/ClickHouse/pull/22242) ([hexiaoting](https://github.com/hexiaoting)). +* Add setting `indexes` (boolean, disabled by default) to `EXPLAIN PIPELINE` query. When enabled, shows used indexes, number of filtered parts and granules for every index applied. Supported for `MergeTree*` tables. [#22352](https://github.com/ClickHouse/ClickHouse/pull/22352) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add setting `json` (boolean, 0 by default) for `EXPLAIN PLAN` query. When enabled, query output will be a single `JSON` row. It is recommended to use `TSVRaw` format to avoid unnecessary escaping. [#23082](https://github.com/ClickHouse/ClickHouse/pull/23082) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Added `SYSTEM QUERY RELOAD MODEL`, `SYSTEM QUERY RELOAD MODELS`. Closes [#18722](https://github.com/ClickHouse/ClickHouse/issues/18722). [#23182](https://github.com/ClickHouse/ClickHouse/pull/23182) ([Maksim Kita](https://github.com/kitaisreal)). +* Made progress bar for LocalServer and united it for Client and Local. [#23196](https://github.com/ClickHouse/ClickHouse/pull/23196) ([Egor Savin](https://github.com/Amesaru)). +* Support DDL dictionaries for DatabaseMemory. Closes [#22354](https://github.com/ClickHouse/ClickHouse/issues/22354). Added support for `DETACH DICTIONARY PERMANENTLY`. Added support for `EXCHANGE DICTIONARIES` for Atomic database engine. Added support for moving dictionaries between databases using `RENAME DICTIONARY`. [#23436](https://github.com/ClickHouse/ClickHouse/pull/23436) ([Maksim Kita](https://github.com/kitaisreal)). +* Allow globs `{...}`, which act like shards, and failover options with separator `|` for URL table function. Closes [#17181](https://github.com/ClickHouse/ClickHouse/issues/17181). [#23446](https://github.com/ClickHouse/ClickHouse/pull/23446) ([Kseniia Sumarokova](https://github.com/kssenii)). +* If `insert_null_as_default` = 1, insert default values instead of NULL in `INSERT ... SELECT` and `INSERT ... SELECT ... UNION ALL ...` queries. Closes [#22832](https://github.com/ClickHouse/ClickHouse/issues/22832). [#23524](https://github.com/ClickHouse/ClickHouse/pull/23524) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Implement table comments. closes [#23225](https://github.com/ClickHouse/ClickHouse/issues/23225). [#23548](https://github.com/ClickHouse/ClickHouse/pull/23548) ([flynn](https://github.com/ucasfl)). +* Introduce a new function: arrayProduct which accept an array as the parameter, and return the product of all the elements in array. Closes [#21613](https://github.com/ClickHouse/ClickHouse/issues/21613). [#23782](https://github.com/ClickHouse/ClickHouse/pull/23782) ([Maksim Kita](https://github.com/kitaisreal)). +* Add postgres-like cast operator (`::`). E.g.: `[1, 2]::Array(UInt8)`, `0.1::Decimal(4, 4)`, `number::UInt16`. [#23871](https://github.com/ClickHouse/ClickHouse/pull/23871) ([Anton Popov](https://github.com/CurtizJ)). +* ... [#23910](https://github.com/ClickHouse/ClickHouse/pull/23910) ([Xiang Zhou](https://github.com/javainthinking)). +* Add function splitByRegexp. [#24077](https://github.com/ClickHouse/ClickHouse/pull/24077) ([abel-cheng](https://github.com/abel-cheng)). +* Add `thread_name` column in `system.stack_trace`. This closes [#23256](https://github.com/ClickHouse/ClickHouse/issues/23256). [#24124](https://github.com/ClickHouse/ClickHouse/pull/24124) ([abel-cheng](https://github.com/abel-cheng)). + +#### Performance Improvement +* Enable `compile_expressions` setting by default. When this setting enabled, compositions of simple functions and operators will be compiled to native code with LLVM at runtime. [#8482](https://github.com/ClickHouse/ClickHouse/pull/8482) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ORC input format reading by stripe instead of reading entire table into memory by once which is cost memory when file size is huge. [#23102](https://github.com/ClickHouse/ClickHouse/pull/23102) ([Chao Ma](https://github.com/godliness)). +* Update `re2` library. Performance of regular expressions matching is improved. Also this PR adds compatibility with gcc-11. [#24196](https://github.com/ClickHouse/ClickHouse/pull/24196) ([Raúl Marín](https://github.com/Algunenano)). + +#### Improvement +* Support Array data type for inserting and selecting data in Arrow, Parquet and ORC formats. [#21770](https://github.com/ClickHouse/ClickHouse/pull/21770) ([taylor12805](https://github.com/taylor12805)). +* Add settings `external_storage_max_read_rows` and `external_storage_max_read_rows` for MySQL table engine, dictionary source and MaterializeMySQL minor data fetches. [#22697](https://github.com/ClickHouse/ClickHouse/pull/22697) ([TCeason](https://github.com/TCeason)). +* Retries on HTTP connection drops in S3. [#22988](https://github.com/ClickHouse/ClickHouse/pull/22988) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix the case when a progress bar in interactive mode in clickhouse-client that appear in the middle of the data may rewrite some parts of visible data in terminal. This closes [#19283](https://github.com/ClickHouse/ClickHouse/issues/19283). [#23050](https://github.com/ClickHouse/ClickHouse/pull/23050) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added possibility to restore MergeTree parts to 'detached' directory for DiskS3. [#23112](https://github.com/ClickHouse/ClickHouse/pull/23112) ([Pavel Kovalenko](https://github.com/Jokser)). +* Skip unavaiable replicas when writing to distributed tables. [#23152](https://github.com/ClickHouse/ClickHouse/pull/23152) ([Amos Bird](https://github.com/amosbird)). +* Support LowCardinality nullability with `join_use_nulls`, close [#15101](https://github.com/ClickHouse/ClickHouse/issues/15101). [#23237](https://github.com/ClickHouse/ClickHouse/pull/23237) ([Vladimir C](https://github.com/vdimir)). +* Disable settings `use_hedged_requests` and `async_socket_for_remote` because there is an evidence that it may cause issues. [#23261](https://github.com/ClickHouse/ClickHouse/pull/23261) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed `quantile(s)TDigest`. Added special handling of singleton centroids according to tdunning/t-digest 3.2+. Also a bug with over-compression of centroids in implementation of earlier version of the algorithm was fixed. [#23314](https://github.com/ClickHouse/ClickHouse/pull/23314) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Allow user to specify empty string instead of database name for `MySQL` storage. Default database will be used for queries. In previous versions it was working for SELECT queries and not support for INSERT was also added. This closes [#19281](https://github.com/ClickHouse/ClickHouse/issues/19281). This can be useful working with `Sphinx` or other MySQL-compatible foreign databases. [#23319](https://github.com/ClickHouse/ClickHouse/pull/23319) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable min_bytes_to_use_mmap_io by default. [#23322](https://github.com/ClickHouse/ClickHouse/pull/23322) ([Azat Khuzhin](https://github.com/azat)). +* If user applied a misconfiguration by mistakenly setting `max_distributed_connections` to value zero, every query to a `Distributed` table will throw exception with a message containing "logical error". But it's really an expected behaviour, not a logical error, so the exception message was slightly incorrect. It also triggered checks in our CI enviroment that ensures that no logical errors ever happen. Instead we will treat `max_distributed_connections` misconfigured to zero as the minimum possible value (one). [#23348](https://github.com/ClickHouse/ClickHouse/pull/23348) ([Azat Khuzhin](https://github.com/azat)). +* Keep default timezone on DateTime operations if it was not provided explicitly. For example, if you add one second to a value of `DateTime` type without timezone it will remain `DateTime` without timezone. In previous versions the value of default timezone was placed to the returned data type explicitly so it becomes DateTime('something'). This closes [#4854](https://github.com/ClickHouse/ClickHouse/issues/4854). [#23392](https://github.com/ClickHouse/ClickHouse/pull/23392) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Previously, MySQL 5.7.9 was not supported due to SQL incompatibility. Now leave MySQL parameter verification to the MaterializeMySQL. [#23413](https://github.com/ClickHouse/ClickHouse/pull/23413) ([TCeason](https://github.com/TCeason)). +* Possibility to change S3 disk settings in runtime via new `SYSTEM RESTART DISK` SQL command. [#23429](https://github.com/ClickHouse/ClickHouse/pull/23429) ([Pavel Kovalenko](https://github.com/Jokser)). +* Respect lock_acquire_timeout_for_background_operations for OPTIMIZE. [#23623](https://github.com/ClickHouse/ClickHouse/pull/23623) ([Azat Khuzhin](https://github.com/azat)). +* Make big integers production ready. Add support for `UInt128` data type. Fix known issues with the `Decimal256` data type. Support big integers in dictionaries. Support `gcd`/`lcm` functions for big integers. Support big integers in array search and conditional functions. Support `LowCardinality(UUID)`. Support big integers in `generateRandom` table function and `clickhouse-obfuscator`. Fix error with returning `UUID` from scalar subqueries. This fixes [#7834](https://github.com/ClickHouse/ClickHouse/issues/7834). This fixes [#23936](https://github.com/ClickHouse/ClickHouse/issues/23936). This fixes [#4176](https://github.com/ClickHouse/ClickHouse/issues/4176). This fixes [#24018](https://github.com/ClickHouse/ClickHouse/issues/24018). This fixes [#17828](https://github.com/ClickHouse/ClickHouse/issues/17828). Backward incompatible change: values of `UUID` type cannot be compared with integer. For example, instead of writing `uuid != 0` type `uuid != '00000000-0000-0000-0000-000000000000'`. [#23631](https://github.com/ClickHouse/ClickHouse/pull/23631) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `_partition_value` virtual column to MergeTree table family. It can be used to prune partition in a deterministic way. It's needed to implement partition matcher for mutations. [#23673](https://github.com/ClickHouse/ClickHouse/pull/23673) ([Amos Bird](https://github.com/amosbird)). +* Enable `async_socket_for_remote` by default. [#23683](https://github.com/ClickHouse/ClickHouse/pull/23683) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* When there is some `ReplicatedMergeTree` tables whose zookeeper is expired, it will throw the error below when we select the meta data of some table from `system.tables` with `select_sequential_consistency` is enabled: `Session expired (Session expired): While executing Tables`. [#23793](https://github.com/ClickHouse/ClickHouse/pull/23793) ([Fuwang Hu](https://github.com/fuwhu)). +* Added `region` parameter for S3 storage and disk. [#23846](https://github.com/ClickHouse/ClickHouse/pull/23846) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Allow configuring different log levels for different logging channels. Closes [#19569](https://github.com/ClickHouse/ClickHouse/issues/19569). [#23857](https://github.com/ClickHouse/ClickHouse/pull/23857) ([filimonov](https://github.com/filimonov)). +* Add `broken_data_files`/`broken_data_compressed_bytes` into `system.distribution_queue`. Add metric for number of files for asynchronous insertion into Distributed tables that has been marked as broken (`BrokenDistributedFilesToInsert`). [#23885](https://github.com/ClickHouse/ClickHouse/pull/23885) ([Azat Khuzhin](https://github.com/azat)). +* Allow to add specific queue settings via table settng `rabbitmq_queue_settings_list`. (Closes [#23737](https://github.com/ClickHouse/ClickHouse/issues/23737) and [#23918](https://github.com/ClickHouse/ClickHouse/issues/23918)). Allow user to control all RabbitMQ setup: if table setting `rabbitmq_queue_consume` is set to `1` - RabbitMQ table engine will only connect to specified queue and will not perform any RabbitMQ consumer-side setup like declaring exchange, queues, bindings. (Closes [#21757](https://github.com/ClickHouse/ClickHouse/issues/21757)). Add proper cleanup when RabbitMQ table is dropped - delete queues, which the table has declared and all bound exchanges - if they were created by the table. [#23887](https://github.com/ClickHouse/ClickHouse/pull/23887) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Measure found rate (the percentage for which the value was found) for dictionaries (see `found_rate` in `system.dictionaries`). [#23916](https://github.com/ClickHouse/ClickHouse/pull/23916) ([Azat Khuzhin](https://github.com/azat)). +* Add hints for Enum names. Closes [#17112](https://github.com/ClickHouse/ClickHouse/issues/17112). [#23919](https://github.com/ClickHouse/ClickHouse/pull/23919) ([flynn](https://github.com/ucasfl)). +* Add support for HTTP compression (determined by `Content-Encoding` HTTP header) in `http` dictionary source. This fixes [#8912](https://github.com/ClickHouse/ClickHouse/issues/8912). [#23946](https://github.com/ClickHouse/ClickHouse/pull/23946) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Preallocate support for hashed/sparse_hashed dictionaries. [#23979](https://github.com/ClickHouse/ClickHouse/pull/23979) ([Azat Khuzhin](https://github.com/azat)). +* Support specifying table schema for postgresql dictionary source. Closes [#23958](https://github.com/ClickHouse/ClickHouse/issues/23958). [#23980](https://github.com/ClickHouse/ClickHouse/pull/23980) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Log information about OS name, kernel version and CPU architecture on server startup. [#23988](https://github.com/ClickHouse/ClickHouse/pull/23988) ([Azat Khuzhin](https://github.com/azat)). +* enable DateTime64 to be a version column in ReplacingMergeTree. [#23992](https://github.com/ClickHouse/ClickHouse/pull/23992) ([kevin wan](https://github.com/MaxWk)). +* Add support for `ORDER BY WITH FILL` with `DateTime64`. [#24016](https://github.com/ClickHouse/ClickHouse/pull/24016) ([kevin wan](https://github.com/MaxWk)). +* Now `prefer_column_name_to_alias = 1` will also favor column names for `group by`, `having` and `order by`. This fixes [#23882](https://github.com/ClickHouse/ClickHouse/issues/23882). [#24022](https://github.com/ClickHouse/ClickHouse/pull/24022) ([Amos Bird](https://github.com/amosbird)). +* Do not acquire lock for total_bytes/total_rows for Buffer engine. [#24066](https://github.com/ClickHouse/ClickHouse/pull/24066) ([Azat Khuzhin](https://github.com/azat)). +* Flush Buffer tables before shutting down tables (within one database), to avoid discarding blocks due to underlying table had been already detached (and `Destination table default.a_data_01870 doesn't exist. Block of data is discarded` error in the log). [#24067](https://github.com/ClickHouse/ClickHouse/pull/24067) ([Azat Khuzhin](https://github.com/azat)). +* Preserve dictionaries until storage shutdown (this will avoid possible `external dictionary 'DICT' not found` errors at server shutdown during final Buffer flush). [#24068](https://github.com/ClickHouse/ClickHouse/pull/24068) ([Azat Khuzhin](https://github.com/azat)). +* Update zstd to v1.5.0. [#24135](https://github.com/ClickHouse/ClickHouse/pull/24135) ([Raúl Marín](https://github.com/Algunenano)). +* Fix crash when memory allocation fails in simdjson. https://github.com/simdjson/simdjson/pull/1567 . Mark as improvement because it's a rare bug. [#24147](https://github.com/ClickHouse/ClickHouse/pull/24147) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix +* This PR fixes a crash on shutdown which happened because of currentConnections() could return zero while some connections were still alive. [#23154](https://github.com/ClickHouse/ClickHouse/pull/23154) ([Vitaly Baranov](https://github.com/vitlibar)). +* QueryAliasVisitor to prefer alias for ASTWithAlias if subquery was optimized to constant. Fixes [#22924](https://github.com/ClickHouse/ClickHouse/issues/22924). Fixes [#10401](https://github.com/ClickHouse/ClickHouse/issues/10401). [#23191](https://github.com/ClickHouse/ClickHouse/pull/23191) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed `Not found column` error when selecting from `MaterializeMySQL` with condition on key column. Fixes [#22432](https://github.com/ClickHouse/ClickHouse/issues/22432). [#23200](https://github.com/ClickHouse/ClickHouse/pull/23200) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed the behavior when disabling `input_format_with_names_use_header ` setting discards all the input with CSVWithNames format. This fixes [#22406](https://github.com/ClickHouse/ClickHouse/issues/22406). [#23202](https://github.com/ClickHouse/ClickHouse/pull/23202) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add type conversion for optimize_skip_unused_shards_rewrite_in (fixes `use-of-uninitialized-value` with `optimize_skip_unused_shards_rewrite_in`). [#23219](https://github.com/ClickHouse/ClickHouse/pull/23219) ([Azat Khuzhin](https://github.com/azat)). +* Fixed simple key dictionary from DDL creation if primary key is not first attribute. Fixes [#23236](https://github.com/ClickHouse/ClickHouse/issues/23236). [#23262](https://github.com/ClickHouse/ClickHouse/pull/23262) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed very rare (distributed) race condition between creation and removal of ReplicatedMergeTree tables. It might cause exceptions like `node doesn't exist` on attempt to create replicated table. Fixes [#21419](https://github.com/ClickHouse/ClickHouse/issues/21419). [#23294](https://github.com/ClickHouse/ClickHouse/pull/23294) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix possible crash in case if `unknown packet` was received form remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Don't relax NOT conditions during partition pruning. This fixes [#23305](https://github.com/ClickHouse/ClickHouse/issues/23305) and [#21539](https://github.com/ClickHouse/ClickHouse/issues/21539). [#23310](https://github.com/ClickHouse/ClickHouse/pull/23310) ([Amos Bird](https://github.com/amosbird)). +* * Fix bug in dict join with join_algorithm = 'auto'. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir C](https://github.com/vdimir)). +* Fix possible `Block structure mismatch` error for queries with `UNION` which could possibly happen after filter-push-down optimization. Fixes [#23029](https://github.com/ClickHouse/ClickHouse/issues/23029). [#23359](https://github.com/ClickHouse/ClickHouse/pull/23359) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix incompatible constant expression generation during partition pruning based on virtual columns. This fixes https://github.com/ClickHouse/ClickHouse/pull/21401#discussion_r611888913. [#23366](https://github.com/ClickHouse/ClickHouse/pull/23366) ([Amos Bird](https://github.com/amosbird)). +* `ORDER BY` with `COLLATE` was not working correctly if the column is in primary key (or is a monotonic function of it) and the setting `optimize_read_in_order` is not turned off. This closes [#22379](https://github.com/ClickHouse/ClickHouse/issues/22379). Workaround for older versions: turn the setting `optimize_read_in_order` off. [#23375](https://github.com/ClickHouse/ClickHouse/pull/23375) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove support for `argMin` and `argMax` for single `Tuple` argument. The code was not memory-safe. The feature was added by mistake and it is confusing for people. These functions can be reintroduced under different names later. This fixes [#22384](https://github.com/ClickHouse/ClickHouse/issues/22384) and reverts [#17359](https://github.com/ClickHouse/ClickHouse/issues/17359). [#23393](https://github.com/ClickHouse/ClickHouse/pull/23393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow to move more conditions to `PREWHERE` as it was before version 21.1. Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)). +* Kafka storage may support parquet format messages. [#23412](https://github.com/ClickHouse/ClickHouse/pull/23412) ([Chao Ma](https://github.com/godliness)). +* Kafka storage may support `arrow` and `arrowstream` format messages. [#23415](https://github.com/ClickHouse/ClickHouse/pull/23415) ([Chao Ma](https://github.com/godliness)). +* Fixed `Cannot unlink file` error on unsuccessful creation of ReplicatedMergeTree table with multidisk configuration. This closes [#21755](https://github.com/ClickHouse/ClickHouse/issues/21755). [#23433](https://github.com/ClickHouse/ClickHouse/pull/23433) ([Alexander Tokmakov](https://github.com/tavplubix)). +* - Bug fix for `deltaSum` aggregate function in counter reset case ... [#23437](https://github.com/ClickHouse/ClickHouse/pull/23437) ([Russ Frank](https://github.com/rf)). +* Fix bug that does not allow cast from empty array literal, to array with dimensions greater than 1. Closes [#14476](https://github.com/ClickHouse/ClickHouse/issues/14476). [#23456](https://github.com/ClickHouse/ClickHouse/pull/23456) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix corner cases in vertical merges with `ReplacingMergeTree`. In rare cases they could lead to fails of merges with exceptions like `Incomplete granules are not allowed while blocks are granules size`. [#23459](https://github.com/ClickHouse/ClickHouse/pull/23459) ([Anton Popov](https://github.com/CurtizJ)). +* When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). +* Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). +* * Fix bug with `Join` and `WITH TOTALS`, close [#17718](https://github.com/ClickHouse/ClickHouse/issues/17718). [#23549](https://github.com/ClickHouse/ClickHouse/pull/23549) ([Vladimir C](https://github.com/vdimir)). +* Fix restart / stop command hanging. Closes [#20214](https://github.com/ClickHouse/ClickHouse/issues/20214). [#23552](https://github.com/ClickHouse/ClickHouse/pull/23552) ([filimonov](https://github.com/filimonov)). +* Fix misinterpretation of some `LIKE` expressions with escape sequences. [#23610](https://github.com/ClickHouse/ClickHouse/pull/23610) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed server fault when inserting data through HTTP caused an exception. This fixes [#23512](https://github.com/ClickHouse/ClickHouse/issues/23512). [#23643](https://github.com/ClickHouse/ClickHouse/pull/23643) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added an exception in case of completely the same values in both samples in aggregate function `mannWhitneyUTest`. This fixes [#23646](https://github.com/ClickHouse/ClickHouse/issues/23646). [#23654](https://github.com/ClickHouse/ClickHouse/pull/23654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed a bug in recovery of staled `ReplicatedMergeTree` replica. Some metadata updates could be ignored by staled replica if `ALTER` query was executed during downtime of the replica. [#23742](https://github.com/ClickHouse/ClickHouse/pull/23742) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Avoid possible "Cannot schedule a task" error (in case some exception had been occurred) on INSERT into Distributed. [#23744](https://github.com/ClickHouse/ClickHouse/pull/23744) ([Azat Khuzhin](https://github.com/azat)). +* Fix `heap_use_after_free` when reading from hdfs if `Values` format is used. [#23761](https://github.com/ClickHouse/ClickHouse/pull/23761) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)). +* Fixed remote JDBC bridge timeout connection issue. Closes [#9609](https://github.com/ClickHouse/ClickHouse/issues/9609). [#23771](https://github.com/ClickHouse/ClickHouse/pull/23771) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix `CLEAR COLUMN` does not work when it is referenced by materialized view. Close [#23764](https://github.com/ClickHouse/ClickHouse/issues/23764). [#23781](https://github.com/ClickHouse/ClickHouse/pull/23781) ([flynn](https://github.com/ucasfl)). +* Fix error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` and `use_hedged_requests`. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better handling of URI's in `PocoHTTPClient`. Fixed bug with URLs containing `+` symbol, data with such keys could not be read previously. [#23822](https://github.com/ClickHouse/ClickHouse/pull/23822) ([Vladimir Chebotarev](https://github.com/excitoon)). +* HashedDictionary complex key update field initial load fix. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)). +* Better handling of HTTP errors in `PocoHTTPClient`. Response bodies of HTTP errors were being ignored earlier. [#23844](https://github.com/ClickHouse/ClickHouse/pull/23844) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix `distributed_group_by_no_merge=2` with `GROUP BY` and aggregate function wrapped into regular function (had been broken in [#23546](https://github.com/ClickHouse/ClickHouse/issues/23546)). Throw exception in case of someone trying to use `distributed_group_by_no_merge=2` with window functions. Disable `optimize_distributed_group_by_sharding_key` for queries with window functions. [#23906](https://github.com/ClickHouse/ClickHouse/pull/23906) ([Azat Khuzhin](https://github.com/azat)). +* Fix implementation of connection pool of PostgreSQL engine. Closes [#23897](https://github.com/ClickHouse/ClickHouse/issues/23897). [#23909](https://github.com/ClickHouse/ClickHouse/pull/23909) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix keys metrics accounting for CACHE() dictionary with duplicates in the source (leads to `DictCacheKeysRequestedMiss` overflows). [#23929](https://github.com/ClickHouse/ClickHouse/pull/23929) ([Azat Khuzhin](https://github.com/azat)). +* Fix SIGSEGV for external GROUP BY and overflow row (i.e. queries like `SELECT FROM GROUP BY WITH TOTALS SETTINGS max_bytes_before_external_group_by>0, max_rows_to_group_by>0, group_by_overflow_mode='any', totals_mode='before_having'`). [#23962](https://github.com/ClickHouse/ClickHouse/pull/23962) ([Azat Khuzhin](https://github.com/azat)). +* Some `ALTER PARTITION` queries might cause `Part A intersects previous part B` and `Unexpected merged part C intersecting drop range D` errors in replication queue. It's fixed. Fixes [#23296](https://github.com/ClickHouse/ClickHouse/issues/23296). [#23997](https://github.com/ClickHouse/ClickHouse/pull/23997) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix crash in MergeJoin, close [#24010](https://github.com/ClickHouse/ClickHouse/issues/24010). [#24013](https://github.com/ClickHouse/ClickHouse/pull/24013) ([Vladimir C](https://github.com/vdimir)). +* now64() supports optional timezone argument ... [#24091](https://github.com/ClickHouse/ClickHouse/pull/24091) ([Vasily Nemkov](https://github.com/Enmk)). +* Fixed using const `DateTime` value vs `DateTime64` column in WHERE. ... [#24100](https://github.com/ClickHouse/ClickHouse/pull/24100) ([Vasily Nemkov](https://github.com/Enmk)). +* Bug: explain pipeline with` select xxx final `shows wrong pipeline: ``` dell123 :) explain pipeline select z from prewhere_move_select_final final;. [#24116](https://github.com/ClickHouse/ClickHouse/pull/24116) ([hexiaoting](https://github.com/hexiaoting)). +* Fix a rare bug that could lead to a partially initialized table that can serve write requests (insert/alter/so on). Now such tables will be in readonly mode. [#24122](https://github.com/ClickHouse/ClickHouse/pull/24122) ([alesapin](https://github.com/alesapin)). +* Fix race condition which could happen in RBAC under a heavy load. This PR fixes [#24090](https://github.com/ClickHouse/ClickHouse/issues/24090), [#24134](https://github.com/ClickHouse/ClickHouse/issues/24134),. [#24176](https://github.com/ClickHouse/ClickHouse/pull/24176) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update nested column with const condition will make server crash. ``` CREATE TABLE test_wide_nested ( `id` Int, `info.id` Array(Int), `info.name` Array(String), `info.age` Array(Int) ) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; set mutations_sync = 1;. [#24183](https://github.com/ClickHouse/ClickHouse/pull/24183) ([hexiaoting](https://github.com/hexiaoting)). +* Fix abnormal server termination due to hdfs becoming not accessible during query execution. Closes [#24117](https://github.com/ClickHouse/ClickHouse/issues/24117). [#24191](https://github.com/ClickHouse/ClickHouse/pull/24191) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix wrong typo at StorageMemory, this bug was introduced at [#15127](https://github.com/ClickHouse/ClickHouse/issues/15127), now fixed, Closes [#24192](https://github.com/ClickHouse/ClickHouse/issues/24192). [#24193](https://github.com/ClickHouse/ClickHouse/pull/24193) ([张中南](https://github.com/plugine)). + +#### Build/Testing/Packaging Improvement +* Adding Map type tests in TestFlows. [#21087](https://github.com/ClickHouse/ClickHouse/pull/21087) ([vzakaznikov](https://github.com/vzakaznikov)). +* Testflows tests for DateTime64 Extended Range. [#22729](https://github.com/ClickHouse/ClickHouse/pull/22729) ([Andrey Zvonov](https://github.com/zvonand)). +* CMake will be failed with settings as bellow ` -DENABLE_CASSANDRA=OFF -DENABLE_AMQPCPP=ON ` ... [#22984](https://github.com/ClickHouse/ClickHouse/pull/22984) ([Ben](https://github.com/benbiti)). +* Add simple tool for benchmarking [Zoo]Keeper. [#23038](https://github.com/ClickHouse/ClickHouse/pull/23038) ([alesapin](https://github.com/alesapin)). +* Remove a source of nondeterminism from build. Now builds at different point of time will produce byte-identical binaries. Partially addressed [#22113](https://github.com/ClickHouse/ClickHouse/issues/22113). [#23559](https://github.com/ClickHouse/ClickHouse/pull/23559) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid possible build dependency on locale and filesystem order. This allows reproducible builds. [#23600](https://github.com/ClickHouse/ClickHouse/pull/23600) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Always enable asynchronous-unwind-tables explicitly. It may fix query profiler on AArch64. [#23602](https://github.com/ClickHouse/ClickHouse/pull/23602) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix Memory Sanitizer report in GRPC library. This closes [#19234](https://github.com/ClickHouse/ClickHouse/issues/19234). [#23615](https://github.com/ClickHouse/ClickHouse/pull/23615) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Window functions tests in TestFlows. [#23704](https://github.com/ClickHouse/ClickHouse/pull/23704) ([vzakaznikov](https://github.com/vzakaznikov)). +* Adds support for building on Solaris-derived operating systems. [#23746](https://github.com/ClickHouse/ClickHouse/pull/23746) ([bnaecker](https://github.com/bnaecker)). +* Update librdkafka 1.6.0-RC3 to 1.6.1. [#23874](https://github.com/ClickHouse/ClickHouse/pull/23874) ([filimonov](https://github.com/filimonov)). +* Enabling running of all TestFlows modules in parallel. [#23942](https://github.com/ClickHouse/ClickHouse/pull/23942) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fixing window functions distributed tests by moving to a deterministic sharding key. [#23975](https://github.com/ClickHouse/ClickHouse/pull/23975) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add more benchmarks for hash tables, including the Swiss Table from Google (that appeared to be slower than ClickHouse hash map in our specific usage scenario). [#24111](https://github.com/ClickHouse/ClickHouse/pull/24111) ([Maksim Kita](https://github.com/kitaisreal)). +* Support building on Illumos. [#24144](https://github.com/ClickHouse/ClickHouse/pull/24144) ([bnaecker](https://github.com/bnaecker)). + +#### Other +* Automated backporting now looks at the label 'release' of PRs to consider the release branch. [#23363](https://github.com/ClickHouse/ClickHouse/pull/23363) ([Ivan](https://github.com/abyss7)). +* Add test cases for arrayElement. related issue: [#22765](https://github.com/ClickHouse/ClickHouse/issues/22765). [#23484](https://github.com/ClickHouse/ClickHouse/pull/23484) ([hexiaoting](https://github.com/hexiaoting)). +* Rename uniqThetaSketch (https://github.com/ClickHouse/ClickHouse/issues/14893) to uniqTheta. [#24019](https://github.com/ClickHouse/ClickHouse/pull/24019) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Function `arrayFold` for folding over array with accumulator"'. [#23248](https://github.com/ClickHouse/ClickHouse/pull/23248) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "[RFC] Fix memory tracking with min_bytes_to_use_mmap_io"'. [#23276](https://github.com/ClickHouse/ClickHouse/pull/23276) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "add uniqThetaSketch"'. [#23334](https://github.com/ClickHouse/ClickHouse/pull/23334) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Fix CI build for gcc-10"'. [#23772](https://github.com/ClickHouse/ClickHouse/pull/23772) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Update syntax.md'. [#24267](https://github.com/ClickHouse/ClickHouse/pull/24267) ([lulichao](https://github.com/lulichao)). + +#### New Feature #14893 + +* - Add uniqThetaSketch to support [Theta Sketch](https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html) in ClickHouse. [#22609](https://github.com/ClickHouse/ClickHouse/pull/22609) ([Ping Yu](https://github.com/pingyu)). +* - Add uniqThetaSketch to support [Theta Sketch](https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html) in ClickHouse. [#23894](https://github.com/ClickHouse/ClickHouse/pull/23894) ([Ping Yu](https://github.com/pingyu)). + diff --git a/docs/changelogs/v21.6.2.7-prestable.md b/docs/changelogs/v21.6.2.7-prestable.md new file mode 100644 index 00000000000..c5dec251786 --- /dev/null +++ b/docs/changelogs/v21.6.2.7-prestable.md @@ -0,0 +1,13 @@ +### ClickHouse release v21.6.2.7-prestable FIXME as compared to v21.6.1.6891-prestable + +#### Bug Fix +* Backported in [#24567](https://github.com/ClickHouse/ClickHouse/issues/24567): Use old modulo function version when used in partition key. Closes [#23508](https://github.com/ClickHouse/ClickHouse/issues/23508). [#24157](https://github.com/ClickHouse/ClickHouse/pull/24157) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#24367](https://github.com/ClickHouse/ClickHouse/issues/24367): Set `max_threads = 1` to fix mutation fail of StorageMemory. Closes [#24274](https://github.com/ClickHouse/ClickHouse/issues/24274). [#24275](https://github.com/ClickHouse/ClickHouse/pull/24275) ([flynn](https://github.com/ucasfl)). +* Backported in [#24383](https://github.com/ClickHouse/ClickHouse/issues/24383): Allow empty HTTP headers. Fixes [#23901](https://github.com/ClickHouse/ClickHouse/issues/23901). [#24285](https://github.com/ClickHouse/ClickHouse/pull/24285) ([Ivan](https://github.com/abyss7)). +* Backported in [#24365](https://github.com/ClickHouse/ClickHouse/issues/24365): Fixed a bug in moving Materialized View from Ordinary to Atomic database (`RENAME TABLE` query). Now inner table is moved to new database together with Materialized View. Fixes [#23926](https://github.com/ClickHouse/ClickHouse/issues/23926). [#24309](https://github.com/ClickHouse/ClickHouse/pull/24309) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24510](https://github.com/ClickHouse/ClickHouse/issues/24510): Fix drop partition with intersect fake parts. In rare cases there might be parts with mutation version greater than current block number. [#24321](https://github.com/ClickHouse/ClickHouse/pull/24321) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#24542](https://github.com/ClickHouse/ClickHouse/issues/24542): Fix incorrect monotonicity of toWeek function. This fixes [#24422](https://github.com/ClickHouse/ClickHouse/issues/24422) . This bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/5212 , and was exposed later by smarter partition pruner. [#24446](https://github.com/ClickHouse/ClickHouse/pull/24446) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#24556](https://github.com/ClickHouse/ClickHouse/issues/24556): Fixed the behavior when query `SYSTEM RESTART REPLICA` or `SYSTEM SYNC REPLICA` is being processed infinitely. This was detected on server with extremely little amount of RAM. [#24457](https://github.com/ClickHouse/ClickHouse/pull/24457) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#24597](https://github.com/ClickHouse/ClickHouse/issues/24597): Fix usage of tuples in `CREATE .. AS SELECT` queries. [#24464](https://github.com/ClickHouse/ClickHouse/pull/24464) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#24600](https://github.com/ClickHouse/ClickHouse/issues/24600): Enable reading of subcolumns for distributed tables. [#24472](https://github.com/ClickHouse/ClickHouse/pull/24472) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.6.3.14-stable.md b/docs/changelogs/v21.6.3.14-stable.md new file mode 100644 index 00000000000..8d2e2a03320 --- /dev/null +++ b/docs/changelogs/v21.6.3.14-stable.md @@ -0,0 +1,14 @@ +### ClickHouse release v21.6.3.14-stable FIXME as compared to v21.6.2.7-prestable + +#### Improvement +* Backported in [#24531](https://github.com/ClickHouse/ClickHouse/issues/24531): Fix Zero-Copy replication with several S3 volumes (Fixes [#22679](https://github.com/ClickHouse/ClickHouse/issues/22679)). [#22864](https://github.com/ClickHouse/ClickHouse/pull/22864) ([ianton-ru](https://github.com/ianton-ru)). +* Backported in [#24776](https://github.com/ClickHouse/ClickHouse/issues/24776): Avoid hiding errors like `Limit for rows or bytes to read exceeded` for scalar subqueries. [#24545](https://github.com/ClickHouse/ClickHouse/pull/24545) ([nvartolomei](https://github.com/nvartolomei)). + +#### Bug Fix +* Backported in [#24727](https://github.com/ClickHouse/ClickHouse/issues/24727): In "multipart/form-data" message consider the CRLF preceding a boundary as part of it. Fixes [#23905](https://github.com/ClickHouse/ClickHouse/issues/23905). [#24399](https://github.com/ClickHouse/ClickHouse/pull/24399) ([Ivan](https://github.com/abyss7)). +* Backported in [#24827](https://github.com/ClickHouse/ClickHouse/issues/24827): - Fixed the deadlock that can happen during LDAP role (re)mapping, when LDAP group is mapped to a nonexistent local role. [#24431](https://github.com/ClickHouse/ClickHouse/pull/24431) ([Denis Glazachev](https://github.com/traceon)). +* Backported in [#24936](https://github.com/ClickHouse/ClickHouse/issues/24936): - If ZooKeeper connection was lost and replica was cloned after restoring the connection, its replication queue might contain outdated entries. It's fixed. - Fixed crash when replication queue contains intersecting virtual parts. It may rarely happen if some data part was lost. Print error in log instead of terminating. [#24777](https://github.com/ClickHouse/ClickHouse/pull/24777) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#24851](https://github.com/ClickHouse/ClickHouse/issues/24851): Fix bug when exception `Mutation was killed` can be thrown to the client on mutation wait when mutation not loaded into memory yet. [#24809](https://github.com/ClickHouse/ClickHouse/pull/24809) ([alesapin](https://github.com/alesapin)). +* Backported in [#24916](https://github.com/ClickHouse/ClickHouse/issues/24916): Fix extremely rare bug on low-memory servers which can lead to the inability to perform merges without restart. Possibly fixes [#24603](https://github.com/ClickHouse/ClickHouse/issues/24603). [#24872](https://github.com/ClickHouse/ClickHouse/pull/24872) ([alesapin](https://github.com/alesapin)). +* Backported in [#24952](https://github.com/ClickHouse/ClickHouse/issues/24952): Fix possible heap-buffer-overflow in Arrow. [#24922](https://github.com/ClickHouse/ClickHouse/pull/24922) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.6.4.26-stable.md b/docs/changelogs/v21.6.4.26-stable.md new file mode 100644 index 00000000000..9bf7d0e68cb --- /dev/null +++ b/docs/changelogs/v21.6.4.26-stable.md @@ -0,0 +1,14 @@ +### ClickHouse release v21.6.4.26-stable FIXME as compared to v21.6.3.14-stable + +#### Bug Fix +* Backported in [#24969](https://github.com/ClickHouse/ClickHouse/issues/24969): Allow NULL values in postgresql protocol. Closes [#22622](https://github.com/ClickHouse/ClickHouse/issues/22622). [#24857](https://github.com/ClickHouse/ClickHouse/pull/24857) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#25186](https://github.com/ClickHouse/ClickHouse/issues/25186): Fixed bug with declaring S3 disk at root of bucket. Earlier, it reported an error: ``` [heather] 2021.05.10 02:11:11.932234 [ 72790 ] {2ff80b7b-ec53-41cb-ac35-19bb390e1759} executeQuery: Code: 36, e.displayText() = DB::Exception: Key name is empty in path style S3 URI: (http://172.17.0.2/bucket/) (version 21.6.1.1) (from 127.0.0.1:47994) (in query: SELECT policy_name FROM system.storage_policies), Stack trace (when copying this message, always include the lines below):. [#24898](https://github.com/ClickHouse/ClickHouse/pull/24898) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#25023](https://github.com/ClickHouse/ClickHouse/issues/25023): Fix limit/offset settings for distributed queries (ignore on the remote nodes). [#24940](https://github.com/ClickHouse/ClickHouse/pull/24940) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#25049](https://github.com/ClickHouse/ClickHouse/issues/25049): Fix extremely rare error `Tagging already tagged part` in replication queue during concurrent `alter move/replace partition`. Possibly fixes [#22142](https://github.com/ClickHouse/ClickHouse/issues/22142). [#24961](https://github.com/ClickHouse/ClickHouse/pull/24961) ([alesapin](https://github.com/alesapin)). +* Backported in [#25081](https://github.com/ClickHouse/ClickHouse/issues/25081): Fix wrong result when using aggregate projection with **not empty** `GROUP BY` key to execute query with `GROUP BY` by **empty** key. [#25055](https://github.com/ClickHouse/ClickHouse/pull/25055) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#25107](https://github.com/ClickHouse/ClickHouse/issues/25107): Fix bug which allows creating tables with columns referencing themselves like `a UInt32 ALIAS a + 1` or `b UInt32 MATERIALIZED b`. Fixes [#24910](https://github.com/ClickHouse/ClickHouse/issues/24910), [#24292](https://github.com/ClickHouse/ClickHouse/issues/24292). [#25059](https://github.com/ClickHouse/ClickHouse/pull/25059) ([alesapin](https://github.com/alesapin)). +* Backported in [#25106](https://github.com/ClickHouse/ClickHouse/issues/25106): Fix bug with constant maps in mapContains that lead to error `empty column was returned by function mapContains`. Closes [#25077](https://github.com/ClickHouse/ClickHouse/issues/25077). [#25080](https://github.com/ClickHouse/ClickHouse/pull/25080) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#25143](https://github.com/ClickHouse/ClickHouse/issues/25143): Fix crash in query with cross join and `joined_subquery_requires_alias = 0`. Fixes [#24011](https://github.com/ClickHouse/ClickHouse/issues/24011). [#25082](https://github.com/ClickHouse/ClickHouse/pull/25082) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25157](https://github.com/ClickHouse/ClickHouse/issues/25157): Fix possible parts loss after updating up to 21.5 in case table used `UUID` in partition key. (It is not recommended to use `UUID` in partition key). Fixes [#25070](https://github.com/ClickHouse/ClickHouse/issues/25070). [#25127](https://github.com/ClickHouse/ClickHouse/pull/25127) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25182](https://github.com/ClickHouse/ClickHouse/issues/25182): Do not use table's projection for `SELECT` with `FINAL`. It is not supported yet. [#25163](https://github.com/ClickHouse/ClickHouse/pull/25163) ([Amos Bird](https://github.com/amosbird)). + diff --git a/docs/changelogs/v21.6.5.37-stable.md b/docs/changelogs/v21.6.5.37-stable.md new file mode 100644 index 00000000000..3d03c31c7e8 --- /dev/null +++ b/docs/changelogs/v21.6.5.37-stable.md @@ -0,0 +1,21 @@ +### ClickHouse release v21.6.5.37-stable FIXME as compared to v21.6.4.26-stable + +#### Improvement +* Backported in [#25221](https://github.com/ClickHouse/ClickHouse/issues/25221): Here will be listed all the bugs that I am gonna to fix in this PR. [#23518](https://github.com/ClickHouse/ClickHouse/pull/23518) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### Bug Fix +* Backported in [#25480](https://github.com/ClickHouse/ClickHouse/issues/25480): Fix "Missing columns" exception when joining Distributed Materialized View. [#24870](https://github.com/ClickHouse/ClickHouse/pull/24870) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#25366](https://github.com/ClickHouse/ClickHouse/issues/25366): Fix serialization of splitted nested messages in Protobuf format. This PR fixes [#24647](https://github.com/ClickHouse/ClickHouse/issues/24647). [#25000](https://github.com/ClickHouse/ClickHouse/pull/25000) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#25102](https://github.com/ClickHouse/ClickHouse/issues/25102): Distinguish KILL MUTATION for different tables (fixes unexpected `Cancelled mutating parts` error). [#25025](https://github.com/ClickHouse/ClickHouse/pull/25025) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#25213](https://github.com/ClickHouse/ClickHouse/issues/25213): Fixed an error which occurred while inserting a subset of columns using CSVWithNames format. Fixes [#25129](https://github.com/ClickHouse/ClickHouse/issues/25129). [#25169](https://github.com/ClickHouse/ClickHouse/pull/25169) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#25352](https://github.com/ClickHouse/ClickHouse/issues/25352): Fix TOCTOU error in installation script. [#25277](https://github.com/ClickHouse/ClickHouse/pull/25277) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#25385](https://github.com/ClickHouse/ClickHouse/issues/25385): Fix incorrect behaviour and UBSan report in big integers. In previous versions `CAST(1e19 AS UInt128)` returned zero. [#25279](https://github.com/ClickHouse/ClickHouse/pull/25279) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#25470](https://github.com/ClickHouse/ClickHouse/issues/25470): Fix joinGetOrNull with not-nullable columns. This fixes [#24261](https://github.com/ClickHouse/ClickHouse/issues/24261). [#25288](https://github.com/ClickHouse/ClickHouse/pull/25288) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#25360](https://github.com/ClickHouse/ClickHouse/issues/25360): Fix error `Bad cast from type DB::ColumnLowCardinality to DB::ColumnVector` for queries where `LowCardinality` argument was used for IN (this bug appeared in 21.6). Fixes [#25187](https://github.com/ClickHouse/ClickHouse/issues/25187). [#25290](https://github.com/ClickHouse/ClickHouse/pull/25290) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25356](https://github.com/ClickHouse/ClickHouse/issues/25356): Fix Logical Error Cannot sum Array/Tuple in min/maxMap. [#25298](https://github.com/ClickHouse/ClickHouse/pull/25298) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#25436](https://github.com/ClickHouse/ClickHouse/issues/25436): Support `SimpleAggregateFunction(LowCardinality)` for `SummingMergeTree`. Fixes [#25134](https://github.com/ClickHouse/ClickHouse/issues/25134). [#25300](https://github.com/ClickHouse/ClickHouse/pull/25300) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25363](https://github.com/ClickHouse/ClickHouse/issues/25363): On ZooKeeper connection loss `ReplicatedMergeTree` table might wait for background operations to complete before trying to reconnect. It's fixed, now background operations are stopped forcefully. [#25306](https://github.com/ClickHouse/ClickHouse/pull/25306) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25388](https://github.com/ClickHouse/ClickHouse/issues/25388): Fix the possibility of non-deterministic behaviour of the `quantileDeterministic` function and similar. This closes [#20480](https://github.com/ClickHouse/ClickHouse/issues/20480). [#25313](https://github.com/ClickHouse/ClickHouse/pull/25313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#25448](https://github.com/ClickHouse/ClickHouse/issues/25448): Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25407](https://github.com/ClickHouse/ClickHouse/issues/25407): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)). + diff --git a/docs/changelogs/v21.6.6.51-stable.md b/docs/changelogs/v21.6.6.51-stable.md new file mode 100644 index 00000000000..55f1fd46119 --- /dev/null +++ b/docs/changelogs/v21.6.6.51-stable.md @@ -0,0 +1,19 @@ +### ClickHouse release v21.6.6.51-stable FIXME as compared to v21.6.5.37-stable + +#### Bug Fix +* Backported in [#25850](https://github.com/ClickHouse/ClickHouse/issues/25850): `CAST` from `Date` to `DateTime` (or `DateTime64`) was not using the timezone of the `DateTime` type. It can also affect the comparison between `Date` and `DateTime`. Inference of the common type for `Date` and `DateTime` also was not using the corresponding timezone. It affected the results of function `if` and array construction. Closes [#24128](https://github.com/ClickHouse/ClickHouse/issues/24128). [#24129](https://github.com/ClickHouse/ClickHouse/pull/24129) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#25677](https://github.com/ClickHouse/ClickHouse/issues/25677): Fixed bug in deserialization of random generator state with might cause some data types such as `AggregateFunction(groupArraySample(N), T))` to behave in a non-deterministic way. [#24538](https://github.com/ClickHouse/ClickHouse/pull/24538) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25535](https://github.com/ClickHouse/ClickHouse/issues/25535): Fixed possible error 'Cannot read from istream at offset 0' when reading a file from DiskS3. [#24885](https://github.com/ClickHouse/ClickHouse/pull/24885) ([Pavel Kovalenko](https://github.com/Jokser)). +* Backported in [#25557](https://github.com/ClickHouse/ClickHouse/issues/25557): Fix potential crash when calculating aggregate function states by aggregation of aggregate function states of other aggregate functions (not a practical use case). See [#24523](https://github.com/ClickHouse/ClickHouse/issues/24523). [#25015](https://github.com/ClickHouse/ClickHouse/pull/25015) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#25506](https://github.com/ClickHouse/ClickHouse/issues/25506): Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Backported in [#25765](https://github.com/ClickHouse/ClickHouse/issues/25765): Fix assertion in PREWHERE with non-uint8 type, close [#19589](https://github.com/ClickHouse/ClickHouse/issues/19589). [#25484](https://github.com/ClickHouse/ClickHouse/pull/25484) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#25635](https://github.com/ClickHouse/ClickHouse/issues/25635): Fix wrong totals for query `WITH TOTALS` and `WITH FILL`. Fixes [#20872](https://github.com/ClickHouse/ClickHouse/issues/20872). [#25539](https://github.com/ClickHouse/ClickHouse/pull/25539) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#25652](https://github.com/ClickHouse/ClickHouse/issues/25652): Fix null pointer dereference in `EXPLAIN AST` without query. [#25631](https://github.com/ClickHouse/ClickHouse/pull/25631) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#25717](https://github.com/ClickHouse/ClickHouse/issues/25717): `REPLACE PARTITION` might be ignored in rare cases if the source partition was empty. It's fixed. Fixes [#24869](https://github.com/ClickHouse/ClickHouse/issues/24869). [#25665](https://github.com/ClickHouse/ClickHouse/pull/25665) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25696](https://github.com/ClickHouse/ClickHouse/issues/25696): Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#25755](https://github.com/ClickHouse/ClickHouse/issues/25755): Fix data race when querying `system.clusters` while reloading the cluster configuration at the same time. [#25737](https://github.com/ClickHouse/ClickHouse/pull/25737) ([Amos Bird](https://github.com/amosbird)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Partial backport [#24061](https://github.com/ClickHouse/ClickHouse/issues/24061) to 21.6'. [#25621](https://github.com/ClickHouse/ClickHouse/pull/25621) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.6.7.57-stable.md b/docs/changelogs/v21.6.7.57-stable.md new file mode 100644 index 00000000000..5ef9026794b --- /dev/null +++ b/docs/changelogs/v21.6.7.57-stable.md @@ -0,0 +1,8 @@ +### ClickHouse release v21.6.7.57-stable FIXME as compared to v21.6.6.51-stable + +#### Bug Fix +* Backported in [#25955](https://github.com/ClickHouse/ClickHouse/issues/25955): Fix `ALTER MODIFY COLUMN` of columns, which participates in TTL expressions. [#25554](https://github.com/ClickHouse/ClickHouse/pull/25554) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#25959](https://github.com/ClickHouse/ClickHouse/issues/25959): Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). +* Backported in [#26096](https://github.com/ClickHouse/ClickHouse/issues/26096): Fix wrong thread estimation for right subquery join in some cases. Close [#24075](https://github.com/ClickHouse/ClickHouse/issues/24075). [#26052](https://github.com/ClickHouse/ClickHouse/pull/26052) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#26143](https://github.com/ClickHouse/ClickHouse/issues/26143): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.6.8.62-stable.md b/docs/changelogs/v21.6.8.62-stable.md new file mode 100644 index 00000000000..1357c762181 --- /dev/null +++ b/docs/changelogs/v21.6.8.62-stable.md @@ -0,0 +1,9 @@ +### ClickHouse release v21.6.8.62-stable FIXME as compared to v21.6.7.57-stable + +#### Bug Fix +* Backported in [#26194](https://github.com/ClickHouse/ClickHouse/issues/26194): Fix sharding_key from column w/o function for remote() (before `select * from remote('127.1', system.one, dummy)` leads to `Unknown column: dummy, there are only columns .` error). [#25824](https://github.com/ClickHouse/ClickHouse/pull/25824) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#26108](https://github.com/ClickHouse/ClickHouse/issues/26108): Fix rare server crash because of `abort` in ZooKeeper client. Fixes [#25813](https://github.com/ClickHouse/ClickHouse/issues/25813). [#26079](https://github.com/ClickHouse/ClickHouse/pull/26079) ([alesapin](https://github.com/alesapin)). +* Backported in [#26167](https://github.com/ClickHouse/ClickHouse/issues/26167): Fix `joinGet` with LowCarinality columns, close [#25993](https://github.com/ClickHouse/ClickHouse/issues/25993). [#26118](https://github.com/ClickHouse/ClickHouse/pull/26118) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#26205](https://github.com/ClickHouse/ClickHouse/issues/26205): Fix potential crash if more than one `untuple` expression is used. [#26179](https://github.com/ClickHouse/ClickHouse/pull/26179) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#26227](https://github.com/ClickHouse/ClickHouse/issues/26227): Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.6.9.7-stable.md b/docs/changelogs/v21.6.9.7-stable.md new file mode 100644 index 00000000000..ecac9dd75ef --- /dev/null +++ b/docs/changelogs/v21.6.9.7-stable.md @@ -0,0 +1,43 @@ +### ClickHouse release v21.6.9.7-stable FIXME as compared to v21.6.8.62-stable + +#### Improvement +* Backported in [#27129](https://github.com/ClickHouse/ClickHouse/issues/27129): If SSDDictionary is created with DDL query, it can be created only inside user_files directory. [#24466](https://github.com/ClickHouse/ClickHouse/pull/24466) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Bug Fix +* Backported in [#26857](https://github.com/ClickHouse/ClickHouse/issues/26857): ParallelFormattingOutputFormat: Use mutex to handle the join to the collector_thread (https://github.com/ClickHouse/ClickHouse/issues/26694). [#26703](https://github.com/ClickHouse/ClickHouse/pull/26703) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#26939](https://github.com/ClickHouse/ClickHouse/issues/26939): Do not remove data on ReplicatedMergeTree table shutdown to avoid creating data to metadata inconsistency. [#26716](https://github.com/ClickHouse/ClickHouse/pull/26716) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#26985](https://github.com/ClickHouse/ClickHouse/issues/26985): Aggregate function parameters might be lost when applying some combinators causing exceptions like `Conversion from AggregateFunction(topKArray, Array(String)) to AggregateFunction(topKArray(10), Array(String)) is not supported`. It's fixed. Fixes [#26196](https://github.com/ClickHouse/ClickHouse/issues/26196) and [#26433](https://github.com/ClickHouse/ClickHouse/issues/26433). [#26814](https://github.com/ClickHouse/ClickHouse/pull/26814) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#26910](https://github.com/ClickHouse/ClickHouse/issues/26910): Fix library-bridge ids load. [#26834](https://github.com/ClickHouse/ClickHouse/pull/26834) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#26946](https://github.com/ClickHouse/ClickHouse/issues/26946): Fix error `Missing columns: 'xxx'` when `DEFAULT` column references other non materialized column without `DEFAULT` expression. Fixes [#26591](https://github.com/ClickHouse/ClickHouse/issues/26591). [#26900](https://github.com/ClickHouse/ClickHouse/pull/26900) ([alesapin](https://github.com/alesapin)). +* Backported in [#27000](https://github.com/ClickHouse/ClickHouse/issues/27000): Fix reading of custom TLDs (stops processing with lower buffer or bigger file). [#26948](https://github.com/ClickHouse/ClickHouse/pull/26948) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27085](https://github.com/ClickHouse/ClickHouse/issues/27085): Now partition ID in queries like `ALTER TABLE ... PARTITION ID xxx` validates for correctness. Fixes [#25718](https://github.com/ClickHouse/ClickHouse/issues/25718). [#26963](https://github.com/ClickHouse/ClickHouse/pull/26963) ([alesapin](https://github.com/alesapin)). +* Backported in [#27052](https://github.com/ClickHouse/ClickHouse/issues/27052): [RFC] Fix possible mutation stack due to race with DROP_RANGE. [#27002](https://github.com/ClickHouse/ClickHouse/pull/27002) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27159](https://github.com/ClickHouse/ClickHouse/issues/27159): Fix synchronization in GRPCServer This PR fixes [#27024](https://github.com/ClickHouse/ClickHouse/issues/27024). [#27064](https://github.com/ClickHouse/ClickHouse/pull/27064) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#27367](https://github.com/ClickHouse/ClickHouse/issues/27367): - Fix uninitialized memory in functions `multiSearch*` with empty array, close [#27169](https://github.com/ClickHouse/ClickHouse/issues/27169). [#27181](https://github.com/ClickHouse/ClickHouse/pull/27181) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#27260](https://github.com/ClickHouse/ClickHouse/issues/27260): In rare cases `system.detached_parts` table might contain incorrect information for some parts, it's fixed. Fixes [#27114](https://github.com/ClickHouse/ClickHouse/issues/27114). [#27183](https://github.com/ClickHouse/ClickHouse/pull/27183) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#27466](https://github.com/ClickHouse/ClickHouse/issues/27466): Fixed incorrect validation of partition id for MergeTree tables that created with old syntax. [#27328](https://github.com/ClickHouse/ClickHouse/pull/27328) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#27646](https://github.com/ClickHouse/ClickHouse/issues/27646): Fix incorrect result for query with row-level security, prewhere and LowCardinality filter. Fixes [#27179](https://github.com/ClickHouse/ClickHouse/issues/27179). [#27329](https://github.com/ClickHouse/ClickHouse/pull/27329) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#27471](https://github.com/ClickHouse/ClickHouse/issues/27471): fix metric BackgroundMessageBrokerSchedulePoolTask, maybe mistyped。. [#27452](https://github.com/ClickHouse/ClickHouse/pull/27452) ([Ben](https://github.com/benbiti)). +* Fixed `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache` configuration parsing. Options `allow_read_expired_keys`, `max_update_queue_size`, `update_queue_push_timeout_milliseconds`, `query_wait_timeout_milliseconds` were not parsed for dictionaries with non `cache` type. [#27523](https://github.com/ClickHouse/ClickHouse/pull/27523) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#27731](https://github.com/ClickHouse/ClickHouse/issues/27731): Fix crash during projection materialization when some parts contain missing columns. This fixes [#27512](https://github.com/ClickHouse/ClickHouse/issues/27512). [#27528](https://github.com/ClickHouse/ClickHouse/pull/27528) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#27978](https://github.com/ClickHouse/ClickHouse/issues/27978): Bugfix for windowFunnel's "strict" mode. This fixes [#27469](https://github.com/ClickHouse/ClickHouse/issues/27469). [#27563](https://github.com/ClickHouse/ClickHouse/pull/27563) ([achimbab](https://github.com/achimbab)). +* Backported in [#27675](https://github.com/ClickHouse/ClickHouse/issues/27675): Fix postgresql table function resulting in non-closing connections. Closes [#26088](https://github.com/ClickHouse/ClickHouse/issues/26088). [#27662](https://github.com/ClickHouse/ClickHouse/pull/27662) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#27699](https://github.com/ClickHouse/ClickHouse/issues/27699): Fix bad type cast when functions like `arrayHas` are applied to arrays of LowCardinality of Nullable of different non-numeric types like `DateTime` and `DateTime64`. In previous versions bad cast occurs. In new version it will lead to exception. This closes [#26330](https://github.com/ClickHouse/ClickHouse/issues/26330). [#27682](https://github.com/ClickHouse/ClickHouse/pull/27682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#27746](https://github.com/ClickHouse/ClickHouse/issues/27746): Remove duplicated source files in CMakeLists.txt in arrow-cmake. [#27736](https://github.com/ClickHouse/ClickHouse/pull/27736) ([李扬](https://github.com/taiyang-li)). +* Backported in [#27865](https://github.com/ClickHouse/ClickHouse/issues/27865): Prevent crashes for some formats when NULL (tombstone) message was coming from Kafka. Closes [#19255](https://github.com/ClickHouse/ClickHouse/issues/19255). [#27794](https://github.com/ClickHouse/ClickHouse/pull/27794) ([filimonov](https://github.com/filimonov)). +* Backported in [#28348](https://github.com/ClickHouse/ClickHouse/issues/28348): Fix a rare bug in `DROP PART` which can lead to the error `Unexpected merged part intersects drop range`. [#27807](https://github.com/ClickHouse/ClickHouse/pull/27807) ([alesapin](https://github.com/alesapin)). +* Backported in [#27957](https://github.com/ClickHouse/ClickHouse/issues/27957): Fix selecting with extremes from a column of the type `LowCardinality(UUID)`. [#27918](https://github.com/ClickHouse/ClickHouse/pull/27918) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#27952](https://github.com/ClickHouse/ClickHouse/issues/27952): Check cluster name before creating Distributed table, do not allow to create a table with incorrect cluster name. Fixes [#27832](https://github.com/ClickHouse/ClickHouse/issues/27832). [#27927](https://github.com/ClickHouse/ClickHouse/pull/27927) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#28206](https://github.com/ClickHouse/ClickHouse/issues/28206): Fix cases, when read buffer fails with 'attempt to read after end of file'. Closes [#26149](https://github.com/ClickHouse/ClickHouse/issues/26149). [#28150](https://github.com/ClickHouse/ClickHouse/pull/28150) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + +#### Build/Testing/Packaging Improvement +* Backported in [#28030](https://github.com/ClickHouse/ClickHouse/issues/28030): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#28119](https://github.com/ClickHouse/ClickHouse/issues/28119): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#28179](https://github.com/ClickHouse/ClickHouse/issues/28179): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). +* Backported in [#28256](https://github.com/ClickHouse/ClickHouse/issues/28256): Fix incorrect behavior in `clickhouse-keeper` when list watches (`getChildren`) triggered with `set` requests for children. [#28190](https://github.com/ClickHouse/ClickHouse/pull/28190) ([alesapin](https://github.com/alesapin)). +* Backported in [#28264](https://github.com/ClickHouse/ClickHouse/issues/28264): Fix possible read of uninitialized memory for queries with `Nullable(LowCardinality)` type and extremes. Fixes [#28165](https://github.com/ClickHouse/ClickHouse/issues/28165). [#28205](https://github.com/ClickHouse/ClickHouse/pull/28205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#28291](https://github.com/ClickHouse/ClickHouse/issues/28291): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.7.1.7283-prestable.md b/docs/changelogs/v21.7.1.7283-prestable.md new file mode 100644 index 00000000000..52de493c4ab --- /dev/null +++ b/docs/changelogs/v21.7.1.7283-prestable.md @@ -0,0 +1,178 @@ +### ClickHouse release v21.7.1.7283-prestable FIXME as compared to v21.6.1.6891-prestable + +#### Backward Incompatible Change +* Improved performance of queries with explicitly defined large sets. Added compatibility setting `legacy_column_name_of_tuple_literal`. It makes sense to set it to `true`, while doing rolling update of cluster from version lower than 21.7 to any higher version. Otherwise distributed queries with explicitly defined sets at `IN` clause may fail during update. [#25371](https://github.com/ClickHouse/ClickHouse/pull/25371) ([Anton Popov](https://github.com/CurtizJ)). +* Forward/backward incompatible change of maximum buffer size in clickhouse-keeper. Better to do it now (before production), than later. [#25421](https://github.com/ClickHouse/ClickHouse/pull/25421) ([alesapin](https://github.com/alesapin)). + +#### New Feature +* Add support for VFS over HDFS. [#11058](https://github.com/ClickHouse/ClickHouse/pull/11058) ([overshov](https://github.com/overshov)). +* Provides a way to restore replicated table when the data is (possibly) present, but the ZooKeeper metadata is lost. Resolves [#13458](https://github.com/ClickHouse/ClickHouse/issues/13458). [#13652](https://github.com/ClickHouse/ClickHouse/pull/13652) ([Mike Kot](https://github.com/myrrc)). +* Implement `sequenceNextNode()` function useful for `flow analysis`. [#19766](https://github.com/ClickHouse/ClickHouse/pull/19766) ([achimbab](https://github.com/achimbab)). +* Added YAML configuration support to configuration loader. This closes [#3607](https://github.com/ClickHouse/ClickHouse/issues/3607). [#21858](https://github.com/ClickHouse/ClickHouse/pull/21858) ([BoloniniD](https://github.com/BoloniniD)). +* Added dateName function. [#23085](https://github.com/ClickHouse/ClickHouse/pull/23085) ([Daniil Kondratyev](https://github.com/dankondr)). +* Add `quantileBFloat16` aggregate function as well as the corresponding `quantilesBFloat16` and `medianBFloat16`. It is very simple and fast quantile estimator with relative error not more than 0.390625%. This closes [#16641](https://github.com/ClickHouse/ClickHouse/issues/16641). [#23204](https://github.com/ClickHouse/ClickHouse/pull/23204) ([Ivan Novitskiy](https://github.com/RedClusive)). +* Support `ALTER DELETE` queries for `Join` table engine. [#23260](https://github.com/ClickHouse/ClickHouse/pull/23260) ([foolchi](https://github.com/foolchi)). +* Add a new boolean setting `prefer_global_in_and_join` which defaults all IN/JOIN as GLOBAL IN/JOIN. [#23434](https://github.com/ClickHouse/ClickHouse/pull/23434) ([Amos Bird](https://github.com/amosbird)). +* add bitpositionToArray function. [#23843](https://github.com/ClickHouse/ClickHouse/pull/23843) ([kevin wan](https://github.com/MaxWk)). +* Add aggregate function `segmentLengthSum`. [#24250](https://github.com/ClickHouse/ClickHouse/pull/24250) ([flynn](https://github.com/ucasfl)). +* Support structs and maps in Arrow/Parquet/ORC and dictionaries in Arrow input/output formats. Present new setting `output_format_arrow_low_cardinality_as_dictionary`. [#24341](https://github.com/ClickHouse/ClickHouse/pull/24341) ([Kruglov Pavel](https://github.com/Avogar)). +* Support `compile_expression` setting for AARCH64. [#24342](https://github.com/ClickHouse/ClickHouse/pull/24342) ([Maksim Kita](https://github.com/kitaisreal)). +* Now clickhouse-keeper supports ZooKeeper-like `digest` ACLs. [#24448](https://github.com/ClickHouse/ClickHouse/pull/24448) ([alesapin](https://github.com/alesapin)). +* Implements the `h3ToGeo` function. [#24867](https://github.com/ClickHouse/ClickHouse/pull/24867) ([Bharat Nallan](https://github.com/bharatnc)). +* Now query_log has two new columns : initial_query_start_time / initial_query_start_time_microsecond that record the starting time of a distributed query if any. [#25022](https://github.com/ClickHouse/ClickHouse/pull/25022) ([Amos Bird](https://github.com/amosbird)). +* Dictionaries added support for Array type. [#25119](https://github.com/ClickHouse/ClickHouse/pull/25119) ([Maksim Kita](https://github.com/kitaisreal)). +* Add `toJSONString` function to serialize columns to their JSON representations. [#25164](https://github.com/ClickHouse/ClickHouse/pull/25164) ([Amos Bird](https://github.com/amosbird)). +* ClickHouse database created with MaterializeMySQL now contains all column comments from the MySQL database that materialized. [#25199](https://github.com/ClickHouse/ClickHouse/pull/25199) ([Storozhuk Kostiantyn](https://github.com/sand6255)). +* Added function `dateName`. Author [Daniil Kondratyev] (@dankondr). [#25372](https://github.com/ClickHouse/ClickHouse/pull/25372) ([Maksim Kita](https://github.com/kitaisreal)). +* Added function `bitPositionsToArray`. Closes [#23792](https://github.com/ClickHouse/ClickHouse/issues/23792). Author [Kevin Wan] (@MaxWk). [#25394](https://github.com/ClickHouse/ClickHouse/pull/25394) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Performance Improvement +* (remove from changelog) Integrate and test experimental compression libraries. Will be available under the flag `allow_experimental_codecs`. This closes [#16775](https://github.com/ClickHouse/ClickHouse/issues/16775). [#17847](https://github.com/ClickHouse/ClickHouse/pull/17847) ([Abi Palagashvili](https://github.com/fibersel)). +* Add exponential backoff to reschedule read attempt in case RabbitMQ queues are empty. Closes [#24340](https://github.com/ClickHouse/ClickHouse/issues/24340). [#24415](https://github.com/ClickHouse/ClickHouse/pull/24415) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Index of type bloom_filter can be used for expressions with `hasAny` function with constant arrays. This closes: [#24291](https://github.com/ClickHouse/ClickHouse/issues/24291). [#24900](https://github.com/ClickHouse/ClickHouse/pull/24900) ([Vasily Nemkov](https://github.com/Enmk)). + +#### Improvement +* Fix Zero-Copy replication with several S3 volumes (Fixes [#22679](https://github.com/ClickHouse/ClickHouse/issues/22679)). [#22864](https://github.com/ClickHouse/ClickHouse/pull/22864) ([ianton-ru](https://github.com/ianton-ru)). +* Add ability to push down LIMIT for distributed queries. [#23027](https://github.com/ClickHouse/ClickHouse/pull/23027) ([Azat Khuzhin](https://github.com/azat)). +* Respect `insert_allow_materialized_columns` (allows materialized columns) for INSERT into `Distributed` table. [#23349](https://github.com/ClickHouse/ClickHouse/pull/23349) ([Azat Khuzhin](https://github.com/azat)). +* Here will be listed all the bugs that I am gonna to fix in this PR. [#23518](https://github.com/ClickHouse/ClickHouse/pull/23518) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Display progress for File table engine in clickhouse-local and on INSERT query in clickhouse-client when data is passed to stdin. Closes [#18209](https://github.com/ClickHouse/ClickHouse/issues/18209). [#23656](https://github.com/ClickHouse/ClickHouse/pull/23656) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Handle column name clashes for storage join, close [#20309](https://github.com/ClickHouse/ClickHouse/issues/20309). [#23769](https://github.com/ClickHouse/ClickHouse/pull/23769) ([Vladimir C](https://github.com/vdimir)). +* Add ability to split distributed batch on failures (i.e. due to memory limits, corruptions), under `distributed_directory_monitor_split_batch_on_failure` (OFF by default). [#23864](https://github.com/ClickHouse/ClickHouse/pull/23864) ([Azat Khuzhin](https://github.com/azat)). +* Add standalone `clickhouse-keeper` symlink to the main `clickhouse` binary. Now it's possible to run coordination without the main clickhouse server. [#24059](https://github.com/ClickHouse/ClickHouse/pull/24059) ([alesapin](https://github.com/alesapin)). +* Suppress exceptions from logger code. [#24069](https://github.com/ClickHouse/ClickHouse/pull/24069) ([Azat Khuzhin](https://github.com/azat)). +* Use global settings for query to `VIEW`. Fixed the behavior when queries to `VIEW` use local settings, that leads to errors if setting on `CREATE VIEW` and `SELECT` were different. As for now, `VIEW` won't use these modified settings, but you can still pass additional settings in `SETTINGS` section of `CREATE VIEW` query. Close [#20551](https://github.com/ClickHouse/ClickHouse/issues/20551). [#24095](https://github.com/ClickHouse/ClickHouse/pull/24095) ([Vladimir C](https://github.com/vdimir)). +* Add settings (`connection_auto_close`/`connection_max_tries`/`connection_pool_size`) for MySQL storage engine. [#24146](https://github.com/ClickHouse/ClickHouse/pull/24146) ([Azat Khuzhin](https://github.com/azat)). +* Fix trailing whitespaces in FROM clause with subqueries in multiline mode, and also changes the output of the queries slightly in a more human friendly way. [#24151](https://github.com/ClickHouse/ClickHouse/pull/24151) ([Azat Khuzhin](https://github.com/azat)). +* Recognize IPv4 addresses like `127.0.1.1` as local. This is controversial and closes [#23504](https://github.com/ClickHouse/ClickHouse/issues/23504). Michael Filimonov will test this feature. [#24316](https://github.com/ClickHouse/ClickHouse/pull/24316) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix IPv6 addresses resolving (i.e. fixes `select * from remote('[::1]', system.one)`). [#24319](https://github.com/ClickHouse/ClickHouse/pull/24319) ([Azat Khuzhin](https://github.com/azat)). +* Now query_log has two new columns : `initial_query_start_time / initial_query_start_time_microsecond` that record the starting time of a distributed query if any. [#24388](https://github.com/ClickHouse/ClickHouse/pull/24388) ([Amos Bird](https://github.com/amosbird)). +* Rewrite more columns to possible alias expressions. This may enable better optimization, such as projections. [#24405](https://github.com/ClickHouse/ClickHouse/pull/24405) ([Amos Bird](https://github.com/amosbird)). +* Added optimization, that transforms some functions to reading of subcolumns to reduce amount of read data. E.g., statement `col IS NULL` is transformed to reading of subcolumn `col.null`. Optimization can be enabled by setting `optimize_functions_to_subcolumns`. [#24406](https://github.com/ClickHouse/ClickHouse/pull/24406) ([Anton Popov](https://github.com/CurtizJ)). +* Fix a data race on Keeper shutdown. [#24412](https://github.com/ClickHouse/ClickHouse/pull/24412) ([alesapin](https://github.com/alesapin)). +* Support postgres schema for insert queries. Closes [#24149](https://github.com/ClickHouse/ClickHouse/issues/24149). [#24413](https://github.com/ClickHouse/ClickHouse/pull/24413) ([Kseniia Sumarokova](https://github.com/kssenii)). +* If SSDDictionary is created with DDL query, it can be created only inside user_files directory. [#24466](https://github.com/ClickHouse/ClickHouse/pull/24466) ([Maksim Kita](https://github.com/kitaisreal)). +* Make String-to-Int parser stricter so that `toInt64('+')` will throw. [#24475](https://github.com/ClickHouse/ClickHouse/pull/24475) ([Amos Bird](https://github.com/amosbird)). +* Add merge tree setting `max_parts_to_merge_at_once` which limits the number of parts that can be merged in the background at once. Doesn't affect `OPTIMIZE FINAL` query. Fixes [#1820](https://github.com/ClickHouse/ClickHouse/issues/1820). [#24496](https://github.com/ClickHouse/ClickHouse/pull/24496) ([alesapin](https://github.com/alesapin)). +* Avoid hiding errors like `Limit for rows or bytes to read exceeded` for scalar subqueries. [#24545](https://github.com/ClickHouse/ClickHouse/pull/24545) ([nvartolomei](https://github.com/nvartolomei)). +* Add two Replicated*MergeTree settings: `max_replicated_fetches_network_bandwidth` and `max_replicated_sends_network_bandwidth` which allows to limit maximum speed of replicated fetches/sends for table. Add two server-wide settings (in `default` user profile): `max_replicated_fetches_network_bandwidth_for_server` and `max_replicated_sends_network_bandwidth_for_server` which limit maximum speed of replication for all tables. The settings are not followed perfectly accurately. Turned off by default. Fixes [#1821](https://github.com/ClickHouse/ClickHouse/issues/1821). [#24573](https://github.com/ClickHouse/ClickHouse/pull/24573) ([alesapin](https://github.com/alesapin)). +* Respect `max_distributed_connections` for `insert_distributed_sync` (otherwise for huge clusters and sync insert it may run out of `max_thread_pool_size`). [#24754](https://github.com/ClickHouse/ClickHouse/pull/24754) ([Azat Khuzhin](https://github.com/azat)). +* Fixed a bug in `Replicated` database engine that might rarely cause some replica to skip enqueued DDL query. [#24805](https://github.com/ClickHouse/ClickHouse/pull/24805) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Some queries require multi-pass semantic analysis. Try reusing built sets for `IN` in this case. [#24874](https://github.com/ClickHouse/ClickHouse/pull/24874) ([Amos Bird](https://github.com/amosbird)). +* Allow `not in` operator to be used in partition pruning. [#24894](https://github.com/ClickHouse/ClickHouse/pull/24894) ([Amos Bird](https://github.com/amosbird)). +* Improved logging of S3 errors, no more double spaces in case of empty keys and buckets. [#24897](https://github.com/ClickHouse/ClickHouse/pull/24897) ([Vladimir Chebotarev](https://github.com/excitoon)). +* For distributed query, when `optimize_skip_unused_shards=1`, allow to skip shard with condition like `(sharding key) IN (one-element-tuple)`. (Tuples with many elements were supported. Tuple with single element did not work because it is parsed as literal). [#24930](https://github.com/ClickHouse/ClickHouse/pull/24930) ([Amos Bird](https://github.com/amosbird)). +* Detect linux version at runtime (for worked nested epoll, that is required for `async_socket_for_remote`/`use_hedged_requests`, otherwise remote queries may stuck). [#25067](https://github.com/ClickHouse/ClickHouse/pull/25067) ([Azat Khuzhin](https://github.com/azat)). +* Increase size of background schedule pool to 128 (`background_schedule_pool_size` setting). It allows avoiding replication queue hung on slow zookeeper connection. [#25072](https://github.com/ClickHouse/ClickHouse/pull/25072) ([alesapin](https://github.com/alesapin)). +* Fix topLevelDomain() for IDN hosts (i.e. `example.рф`), before it returns empty string for such hosts. [#25103](https://github.com/ClickHouse/ClickHouse/pull/25103) ([Azat Khuzhin](https://github.com/azat)). +* On server start, parts with incorrect partition ID would not be ever removed, but always detached. [#25070](https://github.com/ClickHouse/ClickHouse/issues/25070). [#25166](https://github.com/ClickHouse/ClickHouse/pull/25166) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Correct memory tracking in aggregate function `topK`. This closes [#25259](https://github.com/ClickHouse/ClickHouse/issues/25259). [#25260](https://github.com/ClickHouse/ClickHouse/pull/25260) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use separate `clickhouse-bridge` group and user for bridge processes. Set oom_score_adj so the bridges will be first subjects for OOM killer. Set set maximum RSS to 1 GiB. Closes [#23861](https://github.com/ClickHouse/ClickHouse/issues/23861). [#25280](https://github.com/ClickHouse/ClickHouse/pull/25280) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update prompt in `clickhouse-client` and display a message when reconnecting. This closes [#10577](https://github.com/ClickHouse/ClickHouse/issues/10577). [#25281](https://github.com/ClickHouse/ClickHouse/pull/25281) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add support for function `if` with Decimal and Int types on its branches. This closes [#20549](https://github.com/ClickHouse/ClickHouse/issues/20549). This closes [#10142](https://github.com/ClickHouse/ClickHouse/issues/10142). [#25283](https://github.com/ClickHouse/ClickHouse/pull/25283) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add settings `http_max_fields`, `http_max_field_name_size`, `http_max_field_value_size`. [#25296](https://github.com/ClickHouse/ClickHouse/pull/25296) ([Ivan](https://github.com/abyss7)). +* Add == operator on time conditions for sequenceMatch and sequenceCount functions. For eg: sequenceMatch('(?1)(?t==1)(?2)')(time, data = 1, data = 2). [#25299](https://github.com/ClickHouse/ClickHouse/pull/25299) ([Christophe Kalenzaga](https://github.com/mga-chka)). +* Support Interval for LowCardinality, close [#21730](https://github.com/ClickHouse/ClickHouse/issues/21730). [#25410](https://github.com/ClickHouse/ClickHouse/pull/25410) ([Vladimir C](https://github.com/vdimir)). +* Flatbuffers library updated to v.2.0.0. Improvements list https://github.com/google/flatbuffers/releases/tag/v2.0.0. [#25474](https://github.com/ClickHouse/ClickHouse/pull/25474) ([Ilya Yatsishin](https://github.com/qoega)). +* Drop replicas from dirname for internal_replication=true (allows INSERT into Distributed with cluster from any number of replicas, before only 15 replicas was supported, everything more will fail with ENAMETOOLONG while creating directory for async blocks). [#25513](https://github.com/ClickHouse/ClickHouse/pull/25513) ([Azat Khuzhin](https://github.com/azat)). +* Resolve the actual port number bound when a user requests any available port from the operating system. [#25569](https://github.com/ClickHouse/ClickHouse/pull/25569) ([bnaecker](https://github.com/bnaecker)). +* Improve startup time of Distributed engine. [#25663](https://github.com/ClickHouse/ClickHouse/pull/25663) ([Azat Khuzhin](https://github.com/azat)). + +#### Bug Fix +* Fix the bug in failover behavior when Engine=Kafka was not able to start consumption if the same consumer had an empty assignment previously. Closes [#21118](https://github.com/ClickHouse/ClickHouse/issues/21118). [#21267](https://github.com/ClickHouse/ClickHouse/pull/21267) ([filimonov](https://github.com/filimonov)). +* Fix waiting of automatic dropping of empty parts. It could lead to full filling of background pool and stuck of replication. [#23315](https://github.com/ClickHouse/ClickHouse/pull/23315) ([Anton Popov](https://github.com/CurtizJ)). +* Column cardinality in join output same as at the input, close [#23351](https://github.com/ClickHouse/ClickHouse/issues/23351), close [#20315](https://github.com/ClickHouse/ClickHouse/issues/20315). [#24061](https://github.com/ClickHouse/ClickHouse/pull/24061) ([Vladimir C](https://github.com/vdimir)). +* Use old modulo function version when used in partition key. Closes [#23508](https://github.com/ClickHouse/ClickHouse/issues/23508). [#24157](https://github.com/ClickHouse/ClickHouse/pull/24157) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Set `max_threads = 1` to fix mutation fail of StorageMemory. Closes [#24274](https://github.com/ClickHouse/ClickHouse/issues/24274). [#24275](https://github.com/ClickHouse/ClickHouse/pull/24275) ([flynn](https://github.com/ucasfl)). +* Allow empty HTTP headers. Fixes [#23901](https://github.com/ClickHouse/ClickHouse/issues/23901). [#24285](https://github.com/ClickHouse/ClickHouse/pull/24285) ([Ivan](https://github.com/abyss7)). +* Fixed a bug in moving Materialized View from Ordinary to Atomic database (`RENAME TABLE` query). Now inner table is moved to new database together with Materialized View. Fixes [#23926](https://github.com/ClickHouse/ClickHouse/issues/23926). [#24309](https://github.com/ClickHouse/ClickHouse/pull/24309) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix drop partition with intersect fake parts. In rare cases there might be parts with mutation version greater than current block number. [#24321](https://github.com/ClickHouse/ClickHouse/pull/24321) ([Amos Bird](https://github.com/amosbird)). +* In "multipart/form-data" message consider the CRLF preceding a boundary as part of it. Fixes [#23905](https://github.com/ClickHouse/ClickHouse/issues/23905). [#24399](https://github.com/ClickHouse/ClickHouse/pull/24399) ([Ivan](https://github.com/abyss7)). +* - Fixed the deadlock that can happen during LDAP role (re)mapping, when LDAP group is mapped to a nonexistent local role. [#24431](https://github.com/ClickHouse/ClickHouse/pull/24431) ([Denis Glazachev](https://github.com/traceon)). +* Fix incorrect monotonicity of toWeek function. This fixes [#24422](https://github.com/ClickHouse/ClickHouse/issues/24422) . This bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/5212 , and was exposed later by smarter partition pruner. [#24446](https://github.com/ClickHouse/ClickHouse/pull/24446) ([Amos Bird](https://github.com/amosbird)). +* In current CH version total_writes.bytes counter decreases too much during the buffer flush. It leads to counter overflow and totalBytes return something around 17.44 EB some time after the flush. This pr should fix it. ... [#24450](https://github.com/ClickHouse/ClickHouse/pull/24450) ([DimasKovas](https://github.com/DimasKovas)). +* Fixed the behavior when query `SYSTEM RESTART REPLICA` or `SYSTEM SYNC REPLICA` is being processed infinitely. This was detected on server with extremely little amount of RAM. [#24457](https://github.com/ClickHouse/ClickHouse/pull/24457) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix usage of tuples in `CREATE .. AS SELECT` queries. [#24464](https://github.com/ClickHouse/ClickHouse/pull/24464) ([Anton Popov](https://github.com/CurtizJ)). +* Enable reading of subcolumns for distributed tables. [#24472](https://github.com/ClickHouse/ClickHouse/pull/24472) ([Anton Popov](https://github.com/CurtizJ)). +* Disallow building uniqXXXXStates of other aggregation states. [#24523](https://github.com/ClickHouse/ClickHouse/pull/24523) ([Raúl Marín](https://github.com/Algunenano)). +* Fixed bug in deserialization of random generator state with might cause some data types such as `AggregateFunction(groupArraySample(N), T))` to behave in a non-deterministic way. [#24538](https://github.com/ClickHouse/ClickHouse/pull/24538) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix bug which can lead to ZooKeeper client hung inside clickhouse-server. [#24721](https://github.com/ClickHouse/ClickHouse/pull/24721) ([alesapin](https://github.com/alesapin)). +* - If ZooKeeper connection was lost and replica was cloned after restoring the connection, its replication queue might contain outdated entries. It's fixed. - Fixed crash when replication queue contains intersecting virtual parts. It may rarely happen if some data part was lost. Print error in log instead of terminating. [#24777](https://github.com/ClickHouse/ClickHouse/pull/24777) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix bug when exception `Mutation was killed` can be thrown to the client on mutation wait when mutation not loaded into memory yet. [#24809](https://github.com/ClickHouse/ClickHouse/pull/24809) ([alesapin](https://github.com/alesapin)). +* Allow NULL values in postgresql protocol. Closes [#22622](https://github.com/ClickHouse/ClickHouse/issues/22622). [#24857](https://github.com/ClickHouse/ClickHouse/pull/24857) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix "Missing columns" exception when joining Distributed Materialized View. [#24870](https://github.com/ClickHouse/ClickHouse/pull/24870) ([Azat Khuzhin](https://github.com/azat)). +* Fix extremely rare bug on low-memory servers which can lead to the inability to perform merges without restart. Possibly fixes [#24603](https://github.com/ClickHouse/ClickHouse/issues/24603). [#24872](https://github.com/ClickHouse/ClickHouse/pull/24872) ([alesapin](https://github.com/alesapin)). +* Fixed possible error 'Cannot read from istream at offset 0' when reading a file from DiskS3. [#24885](https://github.com/ClickHouse/ClickHouse/pull/24885) ([Pavel Kovalenko](https://github.com/Jokser)). +* Fixed bug with declaring S3 disk at root of bucket. Earlier, it reported an error: ``` [heather] 2021.05.10 02:11:11.932234 [ 72790 ] {2ff80b7b-ec53-41cb-ac35-19bb390e1759} executeQuery: Code: 36, e.displayText() = DB::Exception: Key name is empty in path style S3 URI: (http://172.17.0.2/bucket/) (version 21.6.1.1) (from 127.0.0.1:47994) (in query: SELECT policy_name FROM system.storage_policies), Stack trace (when copying this message, always include the lines below):. [#24898](https://github.com/ClickHouse/ClickHouse/pull/24898) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix possible heap-buffer-overflow in Arrow. [#24922](https://github.com/ClickHouse/ClickHouse/pull/24922) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix limit/offset settings for distributed queries (ignore on the remote nodes). [#24940](https://github.com/ClickHouse/ClickHouse/pull/24940) ([Azat Khuzhin](https://github.com/azat)). +* Fix extremely rare error `Tagging already tagged part` in replication queue during concurrent `alter move/replace partition`. Possibly fixes [#22142](https://github.com/ClickHouse/ClickHouse/issues/22142). [#24961](https://github.com/ClickHouse/ClickHouse/pull/24961) ([alesapin](https://github.com/alesapin)). +* Fix serialization of splitted nested messages in Protobuf format. This PR fixes [#24647](https://github.com/ClickHouse/ClickHouse/issues/24647). [#25000](https://github.com/ClickHouse/ClickHouse/pull/25000) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix potential crash when calculating aggregate function states by aggregation of aggregate function states of other aggregate functions (not a practical use case). See [#24523](https://github.com/ClickHouse/ClickHouse/issues/24523). [#25015](https://github.com/ClickHouse/ClickHouse/pull/25015) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Distinguish KILL MUTATION for different tables (fixes unexpected `Cancelled mutating parts` error). [#25025](https://github.com/ClickHouse/ClickHouse/pull/25025) ([Azat Khuzhin](https://github.com/azat)). +* Fix wrong result when using aggregate projection with **not empty** `GROUP BY` key to execute query with `GROUP BY` by **empty** key. [#25055](https://github.com/ClickHouse/ClickHouse/pull/25055) ([Amos Bird](https://github.com/amosbird)). +* Fix bug which allows creating tables with columns referencing themselves like `a UInt32 ALIAS a + 1` or `b UInt32 MATERIALIZED b`. Fixes [#24910](https://github.com/ClickHouse/ClickHouse/issues/24910), [#24292](https://github.com/ClickHouse/ClickHouse/issues/24292). [#25059](https://github.com/ClickHouse/ClickHouse/pull/25059) ([alesapin](https://github.com/alesapin)). +* Fix bug with constant maps in mapContains that lead to error `empty column was returned by function mapContains`. Closes [#25077](https://github.com/ClickHouse/ClickHouse/issues/25077). [#25080](https://github.com/ClickHouse/ClickHouse/pull/25080) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix crash in query with cross join and `joined_subquery_requires_alias = 0`. Fixes [#24011](https://github.com/ClickHouse/ClickHouse/issues/24011). [#25082](https://github.com/ClickHouse/ClickHouse/pull/25082) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible parts loss after updating up to 21.5 in case table used `UUID` in partition key. (It is not recommended to use `UUID` in partition key). Fixes [#25070](https://github.com/ClickHouse/ClickHouse/issues/25070). [#25127](https://github.com/ClickHouse/ClickHouse/pull/25127) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Do not use table's projection for `SELECT` with `FINAL`. It is not supported yet. [#25163](https://github.com/ClickHouse/ClickHouse/pull/25163) ([Amos Bird](https://github.com/amosbird)). +* Fixed an error which occurred while inserting a subset of columns using CSVWithNames format. Fixes [#25129](https://github.com/ClickHouse/ClickHouse/issues/25129). [#25169](https://github.com/ClickHouse/ClickHouse/pull/25169) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix TOCTOU error in installation script. [#25277](https://github.com/ClickHouse/ClickHouse/pull/25277) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix incorrect behaviour and UBSan report in big integers. In previous versions `CAST(1e19 AS UInt128)` returned zero. [#25279](https://github.com/ClickHouse/ClickHouse/pull/25279) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix joinGetOrNull with not-nullable columns. This fixes [#24261](https://github.com/ClickHouse/ClickHouse/issues/24261). [#25288](https://github.com/ClickHouse/ClickHouse/pull/25288) ([Amos Bird](https://github.com/amosbird)). +* Fix error `Bad cast from type DB::ColumnLowCardinality to DB::ColumnVector` for queries where `LowCardinality` argument was used for IN (this bug appeared in 21.6). Fixes [#25187](https://github.com/ClickHouse/ClickHouse/issues/25187). [#25290](https://github.com/ClickHouse/ClickHouse/pull/25290) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix Logical Error Cannot sum Array/Tuple in min/maxMap. [#25298](https://github.com/ClickHouse/ClickHouse/pull/25298) ([Kruglov Pavel](https://github.com/Avogar)). +* Support `SimpleAggregateFunction(LowCardinality)` for `SummingMergeTree`. Fixes [#25134](https://github.com/ClickHouse/ClickHouse/issues/25134). [#25300](https://github.com/ClickHouse/ClickHouse/pull/25300) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* On ZooKeeper connection loss `ReplicatedMergeTree` table might wait for background operations to complete before trying to reconnect. It's fixed, now background operations are stopped forcefully. [#25306](https://github.com/ClickHouse/ClickHouse/pull/25306) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix the possibility of non-deterministic behaviour of the `quantileDeterministic` function and similar. This closes [#20480](https://github.com/ClickHouse/ClickHouse/issues/20480). [#25313](https://github.com/ClickHouse/ClickHouse/pull/25313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)). +* Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix excessive underscore before the names of the preprocessed configuration files. [#25431](https://github.com/ClickHouse/ClickHouse/pull/25431) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix convertion of datetime with timezone for MySQL, PostgreSQL, ODBC. Closes [#5057](https://github.com/ClickHouse/ClickHouse/issues/5057). [#25528](https://github.com/ClickHouse/ClickHouse/pull/25528) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix segfault in `Arrow` format when using `Decimal256`. Add arrow `Decimal256` support. [#25531](https://github.com/ClickHouse/ClickHouse/pull/25531) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixed case, when sometimes conversion of postgres arrays resulted in String data type, not n-dimensional array, because `attndims` works incorrectly in some cases. Closes [#24804](https://github.com/ClickHouse/ClickHouse/issues/24804). [#25538](https://github.com/ClickHouse/ClickHouse/pull/25538) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix wrong totals for query `WITH TOTALS` and `WITH FILL`. Fixes [#20872](https://github.com/ClickHouse/ClickHouse/issues/20872). [#25539](https://github.com/ClickHouse/ClickHouse/pull/25539) ([Anton Popov](https://github.com/CurtizJ)). +* Fix error `Key expression contains comparison between inconvertible types` for queries with `ARRAY JOIN` in case if array is used in primary key. Fixes [#8247](https://github.com/ClickHouse/ClickHouse/issues/8247). [#25546](https://github.com/ClickHouse/ClickHouse/pull/25546) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bug which can lead to intersecting parts after merges with TTL: `Part all_40_40_0 is covered by all_40_40_1 but should be merged into all_40_41_1. This shouldn't happen often.`. [#25549](https://github.com/ClickHouse/ClickHouse/pull/25549) ([alesapin](https://github.com/alesapin)). +* Fix restore S3 table. [#25601](https://github.com/ClickHouse/ClickHouse/pull/25601) ([ianton-ru](https://github.com/ianton-ru)). +* Fix null pointer dereference in `EXPLAIN AST` without query. [#25631](https://github.com/ClickHouse/ClickHouse/pull/25631) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* `REPLACE PARTITION` might be ignored in rare cases if the source partition was empty. It's fixed. Fixes [#24869](https://github.com/ClickHouse/ClickHouse/issues/24869). [#25665](https://github.com/ClickHouse/ClickHouse/pull/25665) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix mysql select user() return empty. Fixes [#25683](https://github.com/ClickHouse/ClickHouse/issues/25683). [#25697](https://github.com/ClickHouse/ClickHouse/pull/25697) ([sundyli](https://github.com/sundy-li)). +* Fix data race when querying `system.clusters` while reloading the cluster configuration at the same time. [#25737](https://github.com/ClickHouse/ClickHouse/pull/25737) ([Amos Bird](https://github.com/amosbird)). + +#### Build/Testing/Packaging Improvement +* Ubuntu 20.04 is now used to run integration tests, docker-compose version used to run integration tests is updated to 1.28.2. Environment variables now take effect on docker-compose. Rework test_dictionaries_all_layouts_separate_sources to allow parallel run. [#20393](https://github.com/ClickHouse/ClickHouse/pull/20393) ([Ilya Yatsishin](https://github.com/qoega)). +* - Testing for big ints using the following functions: * Arithmetic * Array, tuple, and map * Bit * Comparison * Conversion * Logical * Mathematical * Null * Rounding - Creating a table with columns that use the data types. [#24350](https://github.com/ClickHouse/ClickHouse/pull/24350) ([MyroTk](https://github.com/MyroTk)). +* Add libfuzzer tests for YAMLParser class. [#24480](https://github.com/ClickHouse/ClickHouse/pull/24480) ([BoloniniD](https://github.com/BoloniniD)). +* Adding support to save clickhouse server logs in TestFlows check. [#24504](https://github.com/ClickHouse/ClickHouse/pull/24504) ([vzakaznikov](https://github.com/vzakaznikov)). +* Integration tests configuration has special treatment for dictionaries. Removed remaining dictionaries manual setup. [#24728](https://github.com/ClickHouse/ClickHouse/pull/24728) ([Ilya Yatsishin](https://github.com/qoega)). +* Add integration test cases to cover JDBC bridge. [#25047](https://github.com/ClickHouse/ClickHouse/pull/25047) ([Zhichun Wu](https://github.com/zhicwu)). +* Disabling extended precision data types TestFlows tests. [#25125](https://github.com/ClickHouse/ClickHouse/pull/25125) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fix using Yandex dockerhub registries for TestFlows. [#25133](https://github.com/ClickHouse/ClickHouse/pull/25133) ([vzakaznikov](https://github.com/vzakaznikov)). +* Adding `leadInFrame` and `lagInFrame` window functions TestFlows tests. [#25144](https://github.com/ClickHouse/ClickHouse/pull/25144) ([vzakaznikov](https://github.com/vzakaznikov)). +* Enable build with s3 module in osx [#25217](https://github.com/ClickHouse/ClickHouse/issues/25217). [#25218](https://github.com/ClickHouse/ClickHouse/pull/25218) ([kevin wan](https://github.com/MaxWk)). +* - Added rounding to mathematical and arithmetic function tests for consistent snapshot comparison. - Cleaned up test names so they're more uniform. [#25297](https://github.com/ClickHouse/ClickHouse/pull/25297) ([MyroTk](https://github.com/MyroTk)). +* Increase LDAP verification cooldown performance tests timeout to 600 sec. [#25374](https://github.com/ClickHouse/ClickHouse/pull/25374) ([vzakaznikov](https://github.com/vzakaznikov)). +* Enabling TestFlows RBAC tests. [#25498](https://github.com/ClickHouse/ClickHouse/pull/25498) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add CI check for darwin-aarch64 cross-compilation. [#25560](https://github.com/ClickHouse/ClickHouse/pull/25560) ([Ivan](https://github.com/abyss7)). +* Changed CSS theme to dark for better code highlighting. [#25682](https://github.com/ClickHouse/ClickHouse/pull/25682) ([Mike Kot](https://github.com/myrrc)). + +#### Other +* Introduce ASTTableIdentifier into the code. [#16401](https://github.com/ClickHouse/ClickHouse/pull/16401) ([Ivan](https://github.com/abyss7)). +* Use std::filesystem instad of Poco::File. [#23657](https://github.com/ClickHouse/ClickHouse/pull/23657) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix init script so it does not print 'usage' message for each 'status' command run. [#25046](https://github.com/ClickHouse/ClickHouse/pull/25046) ([Denis Korenevskiy](https://github.com/DenKoren)). +* Fix cron.d task so it does not spam with email messages about current service status. [#25050](https://github.com/ClickHouse/ClickHouse/pull/25050) ([Denis Korenevskiy](https://github.com/DenKoren)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Pass Settings to aggregate function creator"'. [#24524](https://github.com/ClickHouse/ClickHouse/pull/24524) ([Vladimir C](https://github.com/vdimir)). +* NO CL ENTRY: 'Revert "Add initial_query_start_time to query log"'. [#25021](https://github.com/ClickHouse/ClickHouse/pull/25021) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Add run-id option to integration tests"'. [#25526](https://github.com/ClickHouse/ClickHouse/pull/25526) ([alesapin](https://github.com/alesapin)). +* NO CL ENTRY: 'Revert "Implement h3ToGeo function"'. [#25593](https://github.com/ClickHouse/ClickHouse/pull/25593) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### Testing Improvement + +* * Add join related options to stress tests. [#25200](https://github.com/ClickHouse/ClickHouse/pull/25200) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.7.10.4-stable.md b/docs/changelogs/v21.7.10.4-stable.md new file mode 100644 index 00000000000..9056da8ac89 --- /dev/null +++ b/docs/changelogs/v21.7.10.4-stable.md @@ -0,0 +1,19 @@ +### ClickHouse release v21.7.10.4-stable FIXME as compared to v21.7.9.7-stable + +#### Improvement +* Backported in [#28898](https://github.com/ClickHouse/ClickHouse/issues/28898): Use real tmp file instead of predefined "rows_sources" for vertical merges. This avoids generating garbage directories in tmp disks. [#28299](https://github.com/ClickHouse/ClickHouse/pull/28299) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix +* Backported in [#27925](https://github.com/ClickHouse/ClickHouse/issues/27925): Fix PostgreSQL-style cast (`::` operator) with negative numbers. [#27876](https://github.com/ClickHouse/ClickHouse/pull/27876) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28752](https://github.com/ClickHouse/ClickHouse/issues/28752): Fix transformation of disjunctions chain to `IN` (controlled by settings `optimize_min_equality_disjunction_chain_length`) in distributed queries with settings `legacy_column_name_of_tuple_literal = 0`. [#28658](https://github.com/ClickHouse/ClickHouse/pull/28658) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#28509](https://github.com/ClickHouse/ClickHouse/issues/28509): Fixed possible ZooKeeper watches leak on background processing of distributed DDL queue. Closes [#26036](https://github.com/ClickHouse/ClickHouse/issues/26036). [#28446](https://github.com/ClickHouse/ClickHouse/pull/28446) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#28570](https://github.com/ClickHouse/ClickHouse/issues/28570): Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)). +* Backported in [#28598](https://github.com/ClickHouse/ClickHouse/issues/28598): Fix `There is no subcolumn` error, while select from tables, which have `Nested` columns and scalar columns with dot in name and the same prefix as `Nested` (e.g. `n.id UInt32, n.arr1 Array(UInt64), n.arr2 Array(UInt64)`). [#28531](https://github.com/ClickHouse/ClickHouse/pull/28531) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28742](https://github.com/ClickHouse/ClickHouse/issues/28742): Fix the coredump in the creation of distributed tables, when the parameters passed in are wrong. [#28686](https://github.com/ClickHouse/ClickHouse/pull/28686) ([Zhiyong Wang](https://github.com/ljcui)). +* Backported in [#28788](https://github.com/ClickHouse/ClickHouse/issues/28788): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)). +* Backported in [#28947](https://github.com/ClickHouse/ClickHouse/issues/28947): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28931](https://github.com/ClickHouse/ClickHouse/issues/28931): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.7.11.3-stable.md b/docs/changelogs/v21.7.11.3-stable.md new file mode 100644 index 00000000000..66672204713 --- /dev/null +++ b/docs/changelogs/v21.7.11.3-stable.md @@ -0,0 +1,7 @@ +### ClickHouse release v21.7.11.3-stable FIXME as compared to v21.7.10.4-stable + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#29024](https://github.com/ClickHouse/ClickHouse/issues/29024): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29195](https://github.com/ClickHouse/ClickHouse/issues/29195): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.7.2.7-stable.md b/docs/changelogs/v21.7.2.7-stable.md new file mode 100644 index 00000000000..a7af0fed667 --- /dev/null +++ b/docs/changelogs/v21.7.2.7-stable.md @@ -0,0 +1,17 @@ +### ClickHouse release v21.7.2.7-stable FIXME as compared to v21.7.1.7283-prestable + +#### Improvement +* Backported in [#25881](https://github.com/ClickHouse/ClickHouse/issues/25881): Allow to start clickhouse-client with unreadable working directory. [#25817](https://github.com/ClickHouse/ClickHouse/pull/25817) ([ianton-ru](https://github.com/ianton-ru)). + +#### Bug Fix +* Backported in [#25833](https://github.com/ClickHouse/ClickHouse/issues/25833): `CAST` from `Date` to `DateTime` (or `DateTime64`) was not using the timezone of the `DateTime` type. It can also affect the comparison between `Date` and `DateTime`. Inference of the common type for `Date` and `DateTime` also was not using the corresponding timezone. It affected the results of function `if` and array construction. Closes [#24128](https://github.com/ClickHouse/ClickHouse/issues/24128). [#24129](https://github.com/ClickHouse/ClickHouse/pull/24129) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#25766](https://github.com/ClickHouse/ClickHouse/issues/25766): Fix assertion in PREWHERE with non-uint8 type, close [#19589](https://github.com/ClickHouse/ClickHouse/issues/19589). [#25484](https://github.com/ClickHouse/ClickHouse/pull/25484) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#25954](https://github.com/ClickHouse/ClickHouse/issues/25954): Fix `ALTER MODIFY COLUMN` of columns, which participates in TTL expressions. [#25554](https://github.com/ClickHouse/ClickHouse/pull/25554) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#25871](https://github.com/ClickHouse/ClickHouse/issues/25871): Fix rare bug with `DROP PART` query for `ReplicatedMergeTree` tables which can lead to error message `Unexpected merged part intersecting drop range`. [#25783](https://github.com/ClickHouse/ClickHouse/pull/25783) ([alesapin](https://github.com/alesapin)). +* Backported in [#25886](https://github.com/ClickHouse/ClickHouse/issues/25886): Fix ARM exception handling with non default page size. Fixes [#25512](https://github.com/ClickHouse/ClickHouse/issues/25512). Fixes [#25044](https://github.com/ClickHouse/ClickHouse/issues/25044). Fixes [#24901](https://github.com/ClickHouse/ClickHouse/issues/24901). Fixes [#23183](https://github.com/ClickHouse/ClickHouse/issues/23183). Fixes [#20221](https://github.com/ClickHouse/ClickHouse/issues/20221). Fixes [#19703](https://github.com/ClickHouse/ClickHouse/issues/19703). Fixes [#19028](https://github.com/ClickHouse/ClickHouse/issues/19028). Fixes [#18391](https://github.com/ClickHouse/ClickHouse/issues/18391). Fixes [#18121](https://github.com/ClickHouse/ClickHouse/issues/18121). Fixes [#17994](https://github.com/ClickHouse/ClickHouse/issues/17994). Fixes [#12483](https://github.com/ClickHouse/ClickHouse/issues/12483). [#25854](https://github.com/ClickHouse/ClickHouse/pull/25854) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#25957](https://github.com/ClickHouse/ClickHouse/issues/25957): Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). +* Backported in [#25932](https://github.com/ClickHouse/ClickHouse/issues/25932): Fix crash on call dictGet() with bad arguments. [#25913](https://github.com/ClickHouse/ClickHouse/pull/25913) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#25981](https://github.com/ClickHouse/ClickHouse/issues/25981): Fix possible deadlock during query profiler stack unwinding. Fixes [#25968](https://github.com/ClickHouse/ClickHouse/issues/25968). [#25970](https://github.com/ClickHouse/ClickHouse/pull/25970) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#26010](https://github.com/ClickHouse/ClickHouse/issues/26010): Fix formatting of type `Map` with integer keys to `JSON`. [#25982](https://github.com/ClickHouse/ClickHouse/pull/25982) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#26097](https://github.com/ClickHouse/ClickHouse/issues/26097): Fix wrong thread estimation for right subquery join in some cases. Close [#24075](https://github.com/ClickHouse/ClickHouse/issues/24075). [#26052](https://github.com/ClickHouse/ClickHouse/pull/26052) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.7.3.14-stable.md b/docs/changelogs/v21.7.3.14-stable.md new file mode 100644 index 00000000000..d24b7bcbf39 --- /dev/null +++ b/docs/changelogs/v21.7.3.14-stable.md @@ -0,0 +1,11 @@ +### ClickHouse release v21.7.3.14-stable FIXME as compared to v21.7.2.7-stable + +#### Bug Fix +* Backported in [#26191](https://github.com/ClickHouse/ClickHouse/issues/26191): Fix sharding_key from column w/o function for remote() (before `select * from remote('127.1', system.one, dummy)` leads to `Unknown column: dummy, there are only columns .` error). [#25824](https://github.com/ClickHouse/ClickHouse/pull/25824) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#26109](https://github.com/ClickHouse/ClickHouse/issues/26109): Fix rare server crash because of `abort` in ZooKeeper client. Fixes [#25813](https://github.com/ClickHouse/ClickHouse/issues/25813). [#26079](https://github.com/ClickHouse/ClickHouse/pull/26079) ([alesapin](https://github.com/alesapin)). +* Backported in [#26142](https://github.com/ClickHouse/ClickHouse/issues/26142): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#26170](https://github.com/ClickHouse/ClickHouse/issues/26170): Fix `joinGet` with LowCarinality columns, close [#25993](https://github.com/ClickHouse/ClickHouse/issues/25993). [#26118](https://github.com/ClickHouse/ClickHouse/pull/26118) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#26206](https://github.com/ClickHouse/ClickHouse/issues/26206): Fix potential crash if more than one `untuple` expression is used. [#26179](https://github.com/ClickHouse/ClickHouse/pull/26179) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#26229](https://github.com/ClickHouse/ClickHouse/issues/26229): Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix throwing exception when iterate over non existing remote directory. [#26296](https://github.com/ClickHouse/ClickHouse/pull/26296) ([ianton-ru](https://github.com/ianton-ru)). + diff --git a/docs/changelogs/v21.7.4.18-stable.md b/docs/changelogs/v21.7.4.18-stable.md new file mode 100644 index 00000000000..7bc08e2a0e3 --- /dev/null +++ b/docs/changelogs/v21.7.4.18-stable.md @@ -0,0 +1,10 @@ +### ClickHouse release v21.7.4.18-stable FIXME as compared to v21.7.3.14-stable + +#### Bug Fix +* Backported in [#26297](https://github.com/ClickHouse/ClickHouse/issues/26297): Fixed incorrect `sequence_id` in MySQL protocol packets that ClickHouse sends on exception during query execution. It might cause MySQL client to reset connection to ClickHouse server. Fixes [#21184](https://github.com/ClickHouse/ClickHouse/issues/21184). [#26051](https://github.com/ClickHouse/ClickHouse/pull/26051) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#26356](https://github.com/ClickHouse/ClickHouse/issues/26356): Fix logical error on join with totals, close [#26017](https://github.com/ClickHouse/ClickHouse/issues/26017). [#26250](https://github.com/ClickHouse/ClickHouse/pull/26250) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#26359](https://github.com/ClickHouse/ClickHouse/issues/26359): Fixed rare bug in lost replica recovery that may cause replicas to diverge. [#26321](https://github.com/ClickHouse/ClickHouse/pull/26321) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#26419](https://github.com/ClickHouse/ClickHouse/issues/26419): Fix possible crash when login as dropped user. This PR fixes [#26073](https://github.com/ClickHouse/ClickHouse/issues/26073). [#26363](https://github.com/ClickHouse/ClickHouse/pull/26363) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#26413](https://github.com/ClickHouse/ClickHouse/issues/26413): Fix infinite non joined block stream in `partial_merge_join` close [#26325](https://github.com/ClickHouse/ClickHouse/issues/26325). [#26374](https://github.com/ClickHouse/ClickHouse/pull/26374) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#26447](https://github.com/ClickHouse/ClickHouse/issues/26447): Fix some fuzzed msan crash. Fixes [#22517](https://github.com/ClickHouse/ClickHouse/issues/22517). [#26428](https://github.com/ClickHouse/ClickHouse/pull/26428) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.7.5.29-stable.md b/docs/changelogs/v21.7.5.29-stable.md new file mode 100644 index 00000000000..3f24e3eded9 --- /dev/null +++ b/docs/changelogs/v21.7.5.29-stable.md @@ -0,0 +1,16 @@ +### ClickHouse release v21.7.5.29-stable FIXME as compared to v21.7.4.18-stable + +#### Performance Improvement +* Backported in [#26526](https://github.com/ClickHouse/ClickHouse/issues/26526): Improve latency of short queries, that require reading from tables with large number of columns. [#26371](https://github.com/ClickHouse/ClickHouse/pull/26371) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix +* Backported in [#26787](https://github.com/ClickHouse/ClickHouse/issues/26787): Fix zstd decompression in case there are escape sequences at the end of internal buffer. Closes [#26013](https://github.com/ClickHouse/ClickHouse/issues/26013). [#26314](https://github.com/ClickHouse/ClickHouse/pull/26314) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#26487](https://github.com/ClickHouse/ClickHouse/issues/26487): Fix broken name resolution after rewriting column aliases. This fixes [#26432](https://github.com/ClickHouse/ClickHouse/issues/26432). [#26475](https://github.com/ClickHouse/ClickHouse/pull/26475) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#26615](https://github.com/ClickHouse/ClickHouse/issues/26615): Fix issues with `CREATE DICTIONARY` query if dictionary name or database name was quoted. Closes [#26491](https://github.com/ClickHouse/ClickHouse/issues/26491). [#26508](https://github.com/ClickHouse/ClickHouse/pull/26508) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#26606](https://github.com/ClickHouse/ClickHouse/issues/26606): Fix crash in rabbitmq shutdown in case rabbitmq setup was not started. Closes [#26504](https://github.com/ClickHouse/ClickHouse/issues/26504). [#26529](https://github.com/ClickHouse/ClickHouse/pull/26529) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#26612](https://github.com/ClickHouse/ClickHouse/issues/26612): Update `chown` cmd check in clickhouse-server docker entrypoint. It fixes the bug that cluster pod restart failed (or timeout) on kubernetes. [#26545](https://github.com/ClickHouse/ClickHouse/pull/26545) ([Ky Li](https://github.com/Kylinrix)). +* Backported in [#26647](https://github.com/ClickHouse/ClickHouse/issues/26647): Fix incorrect function names of groupBitmapAnd/Or/Xor. This fixes. [#26557](https://github.com/ClickHouse/ClickHouse/pull/26557) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#26704](https://github.com/ClickHouse/ClickHouse/issues/26704): Fix potential nullptr dereference in window functions. This fixes [#25276](https://github.com/ClickHouse/ClickHouse/issues/25276). [#26668](https://github.com/ClickHouse/ClickHouse/pull/26668) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#26772](https://github.com/ClickHouse/ClickHouse/issues/26772): Sometimes SET ROLE could work incorrectly, this PR fixes that. [#26707](https://github.com/ClickHouse/ClickHouse/pull/26707) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#26907](https://github.com/ClickHouse/ClickHouse/issues/26907): Fix library-bridge ids load. [#26834](https://github.com/ClickHouse/ClickHouse/pull/26834) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.7.6.39-stable.md b/docs/changelogs/v21.7.6.39-stable.md new file mode 100644 index 00000000000..a7913aca193 --- /dev/null +++ b/docs/changelogs/v21.7.6.39-stable.md @@ -0,0 +1,14 @@ +### ClickHouse release v21.7.6.39-stable FIXME as compared to v21.7.5.29-stable + +#### Bug Fix +* Backported in [#26854](https://github.com/ClickHouse/ClickHouse/issues/26854): ParallelFormattingOutputFormat: Use mutex to handle the join to the collector_thread (https://github.com/ClickHouse/ClickHouse/issues/26694). [#26703](https://github.com/ClickHouse/ClickHouse/pull/26703) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#26938](https://github.com/ClickHouse/ClickHouse/issues/26938): Do not remove data on ReplicatedMergeTree table shutdown to avoid creating data to metadata inconsistency. [#26716](https://github.com/ClickHouse/ClickHouse/pull/26716) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#26986](https://github.com/ClickHouse/ClickHouse/issues/26986): Aggregate function parameters might be lost when applying some combinators causing exceptions like `Conversion from AggregateFunction(topKArray, Array(String)) to AggregateFunction(topKArray(10), Array(String)) is not supported`. It's fixed. Fixes [#26196](https://github.com/ClickHouse/ClickHouse/issues/26196) and [#26433](https://github.com/ClickHouse/ClickHouse/issues/26433). [#26814](https://github.com/ClickHouse/ClickHouse/pull/26814) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#26944](https://github.com/ClickHouse/ClickHouse/issues/26944): Fix error `Missing columns: 'xxx'` when `DEFAULT` column references other non materialized column without `DEFAULT` expression. Fixes [#26591](https://github.com/ClickHouse/ClickHouse/issues/26591). [#26900](https://github.com/ClickHouse/ClickHouse/pull/26900) ([alesapin](https://github.com/alesapin)). +* Backported in [#26999](https://github.com/ClickHouse/ClickHouse/issues/26999): Fix reading of custom TLDs (stops processing with lower buffer or bigger file). [#26948](https://github.com/ClickHouse/ClickHouse/pull/26948) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27029](https://github.com/ClickHouse/ClickHouse/issues/27029): Now partition ID in queries like `ALTER TABLE ... PARTITION ID xxx` validates for correctness. Fixes [#25718](https://github.com/ClickHouse/ClickHouse/issues/25718). [#26963](https://github.com/ClickHouse/ClickHouse/pull/26963) ([alesapin](https://github.com/alesapin)). +* Backported in [#27050](https://github.com/ClickHouse/ClickHouse/issues/27050): [RFC] Fix possible mutation stack due to race with DROP_RANGE. [#27002](https://github.com/ClickHouse/ClickHouse/pull/27002) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27105](https://github.com/ClickHouse/ClickHouse/issues/27105): Fixed `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache` configuration parsing. Options `allow_read_expired_keys`, `max_update_queue_size`, `update_queue_push_timeout_milliseconds`, `query_wait_timeout_milliseconds` were not parsed for dictionaries with non `cache` type. [#27032](https://github.com/ClickHouse/ClickHouse/pull/27032) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#27156](https://github.com/ClickHouse/ClickHouse/issues/27156): Fix synchronization in GRPCServer This PR fixes [#27024](https://github.com/ClickHouse/ClickHouse/issues/27024). [#27064](https://github.com/ClickHouse/ClickHouse/pull/27064) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#27261](https://github.com/ClickHouse/ClickHouse/issues/27261): In rare cases `system.detached_parts` table might contain incorrect information for some parts, it's fixed. Fixes [#27114](https://github.com/ClickHouse/ClickHouse/issues/27114). [#27183](https://github.com/ClickHouse/ClickHouse/pull/27183) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.7.7.47-stable.md b/docs/changelogs/v21.7.7.47-stable.md new file mode 100644 index 00000000000..f81abca2600 --- /dev/null +++ b/docs/changelogs/v21.7.7.47-stable.md @@ -0,0 +1,8 @@ +### ClickHouse release v21.7.7.47-stable FIXME as compared to v21.7.6.39-stable + +#### Bug Fix +* Backported in [#27364](https://github.com/ClickHouse/ClickHouse/issues/27364): - Fix uninitialized memory in functions `multiSearch*` with empty array, close [#27169](https://github.com/ClickHouse/ClickHouse/issues/27169). [#27181](https://github.com/ClickHouse/ClickHouse/pull/27181) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#27412](https://github.com/ClickHouse/ClickHouse/issues/27412): Fix `distributed_group_by_no_merge=2`+`distributed_push_down_limit=1` or `optimize_distributed_group_by_sharding_key=1` with `LIMIT BY` and `LIMIT OFFSET`. [#27249](https://github.com/ClickHouse/ClickHouse/pull/27249) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27418](https://github.com/ClickHouse/ClickHouse/issues/27418): Fix `Cannot find column` error for queries with sampling. Was introduced in [#24574](https://github.com/ClickHouse/ClickHouse/issues/24574). Fixes [#26522](https://github.com/ClickHouse/ClickHouse/issues/26522). [#27301](https://github.com/ClickHouse/ClickHouse/pull/27301) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#27416](https://github.com/ClickHouse/ClickHouse/issues/27416): Fixed incorrect validation of partition id for MergeTree tables that created with old syntax. [#27328](https://github.com/ClickHouse/ClickHouse/pull/27328) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.7.8.58-stable.md b/docs/changelogs/v21.7.8.58-stable.md new file mode 100644 index 00000000000..aea1ae083f0 --- /dev/null +++ b/docs/changelogs/v21.7.8.58-stable.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.7.8.58-stable FIXME as compared to v21.7.7.47-stable + +#### Bug Fix +* Backported in [#27506](https://github.com/ClickHouse/ClickHouse/issues/27506): Fix errors like `Expected ColumnLowCardinality, gotUInt8` or `Bad cast from type DB::ColumnVector to DB::ColumnLowCardinality` for some queries with `LowCardinality` in `PREWHERE`. Fixes [#23515](https://github.com/ClickHouse/ClickHouse/issues/23515). [#27298](https://github.com/ClickHouse/ClickHouse/pull/27298) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#27644](https://github.com/ClickHouse/ClickHouse/issues/27644): Fix incorrect result for query with row-level security, prewhere and LowCardinality filter. Fixes [#27179](https://github.com/ClickHouse/ClickHouse/issues/27179). [#27329](https://github.com/ClickHouse/ClickHouse/pull/27329) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#27474](https://github.com/ClickHouse/ClickHouse/issues/27474): fix metric BackgroundMessageBrokerSchedulePoolTask, maybe mistyped。. [#27452](https://github.com/ClickHouse/ClickHouse/pull/27452) ([Ben](https://github.com/benbiti)). +* Backported in [#27649](https://github.com/ClickHouse/ClickHouse/issues/27649): Fix crash during projection materialization when some parts contain missing columns. This fixes [#27512](https://github.com/ClickHouse/ClickHouse/issues/27512). [#27528](https://github.com/ClickHouse/ClickHouse/pull/27528) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#27778](https://github.com/ClickHouse/ClickHouse/issues/27778): - Fix bug with aliased column in `Distributed` table. [#27652](https://github.com/ClickHouse/ClickHouse/pull/27652) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#27674](https://github.com/ClickHouse/ClickHouse/issues/27674): Fix postgresql table function resulting in non-closing connections. Closes [#26088](https://github.com/ClickHouse/ClickHouse/issues/26088). [#27662](https://github.com/ClickHouse/ClickHouse/pull/27662) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#27697](https://github.com/ClickHouse/ClickHouse/issues/27697): Fix bad type cast when functions like `arrayHas` are applied to arrays of LowCardinality of Nullable of different non-numeric types like `DateTime` and `DateTime64`. In previous versions bad cast occurs. In new version it will lead to exception. This closes [#26330](https://github.com/ClickHouse/ClickHouse/issues/26330). [#27682](https://github.com/ClickHouse/ClickHouse/pull/27682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#27748](https://github.com/ClickHouse/ClickHouse/issues/27748): Remove duplicated source files in CMakeLists.txt in arrow-cmake. [#27736](https://github.com/ClickHouse/ClickHouse/pull/27736) ([李扬](https://github.com/taiyang-li)). + diff --git a/docs/changelogs/v21.7.9.7-stable.md b/docs/changelogs/v21.7.9.7-stable.md new file mode 100644 index 00000000000..0d1a2a521e7 --- /dev/null +++ b/docs/changelogs/v21.7.9.7-stable.md @@ -0,0 +1,27 @@ +### ClickHouse release v21.7.9.7-stable FIXME as compared to v21.7.8.58-stable + +#### Improvement +* Backported in [#27892](https://github.com/ClickHouse/ClickHouse/issues/27892): Allow symlinks for library dictionaty path. [#27815](https://github.com/ClickHouse/ClickHouse/pull/27815) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#28155](https://github.com/ClickHouse/ClickHouse/issues/28155): Use Multipart copy upload for large S3 objects. [#27858](https://github.com/ClickHouse/ClickHouse/pull/27858) ([ianton-ru](https://github.com/ianton-ru)). + +#### Bug Fix +* Backported in [#27976](https://github.com/ClickHouse/ClickHouse/issues/27976): Bugfix for windowFunnel's "strict" mode. This fixes [#27469](https://github.com/ClickHouse/ClickHouse/issues/27469). [#27563](https://github.com/ClickHouse/ClickHouse/pull/27563) ([achimbab](https://github.com/achimbab)). +* Backported in [#27780](https://github.com/ClickHouse/ClickHouse/issues/27780): Fix column filtering with union distinct in subquery. Closes [#27578](https://github.com/ClickHouse/ClickHouse/issues/27578). [#27689](https://github.com/ClickHouse/ClickHouse/pull/27689) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#27866](https://github.com/ClickHouse/ClickHouse/issues/27866): Prevent crashes for some formats when NULL (tombstone) message was coming from Kafka. Closes [#19255](https://github.com/ClickHouse/ClickHouse/issues/19255). [#27794](https://github.com/ClickHouse/ClickHouse/pull/27794) ([filimonov](https://github.com/filimonov)). +* Backported in [#28347](https://github.com/ClickHouse/ClickHouse/issues/28347): Fix a rare bug in `DROP PART` which can lead to the error `Unexpected merged part intersects drop range`. [#27807](https://github.com/ClickHouse/ClickHouse/pull/27807) ([alesapin](https://github.com/alesapin)). +* Backported in [#27955](https://github.com/ClickHouse/ClickHouse/issues/27955): Fix selecting with extremes from a column of the type `LowCardinality(UUID)`. [#27918](https://github.com/ClickHouse/ClickHouse/pull/27918) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#27951](https://github.com/ClickHouse/ClickHouse/issues/27951): Check cluster name before creating Distributed table, do not allow to create a table with incorrect cluster name. Fixes [#27832](https://github.com/ClickHouse/ClickHouse/issues/27832). [#27927](https://github.com/ClickHouse/ClickHouse/pull/27927) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#28208](https://github.com/ClickHouse/ClickHouse/issues/28208): Fix cases, when read buffer fails with 'attempt to read after end of file'. Closes [#26149](https://github.com/ClickHouse/ClickHouse/issues/26149). [#28150](https://github.com/ClickHouse/ClickHouse/pull/28150) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + +#### Build/Testing/Packaging Improvement +* Backported in [#28032](https://github.com/ClickHouse/ClickHouse/issues/28032): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#28116](https://github.com/ClickHouse/ClickHouse/issues/28116): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#28183](https://github.com/ClickHouse/ClickHouse/issues/28183): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). +* Backported in [#28255](https://github.com/ClickHouse/ClickHouse/issues/28255): Fix incorrect behavior in `clickhouse-keeper` when list watches (`getChildren`) triggered with `set` requests for children. [#28190](https://github.com/ClickHouse/ClickHouse/pull/28190) ([alesapin](https://github.com/alesapin)). +* Backported in [#28265](https://github.com/ClickHouse/ClickHouse/issues/28265): Fix possible read of uninitialized memory for queries with `Nullable(LowCardinality)` type and extremes. Fixes [#28165](https://github.com/ClickHouse/ClickHouse/issues/28165). [#28205](https://github.com/ClickHouse/ClickHouse/pull/28205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#28288](https://github.com/ClickHouse/ClickHouse/issues/28288): Fix reading of custom TLD w/o new line at EOF. [#28213](https://github.com/ClickHouse/ClickHouse/pull/28213) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#28295](https://github.com/ClickHouse/ClickHouse/issues/28295): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.8.1.7409-prestable.md b/docs/changelogs/v21.8.1.7409-prestable.md new file mode 100644 index 00000000000..e703d227603 --- /dev/null +++ b/docs/changelogs/v21.8.1.7409-prestable.md @@ -0,0 +1,88 @@ +### ClickHouse release v21.8.1.7409-prestable FIXME as compared to v21.7.1.7283-prestable + +#### Backward Incompatible Change +* - Backward Incompatible Change:. [#23934](https://github.com/ClickHouse/ClickHouse/pull/23934) ([hexiaoting](https://github.com/hexiaoting)). + +#### New Feature +* Add an ability to reset custom setting to default and remove it from table's metadata. This will allow to rollback the change without knowing the system/config's default. Closes [#14449](https://github.com/ClickHouse/ClickHouse/issues/14449). [#17769](https://github.com/ClickHouse/ClickHouse/pull/17769) ([xjewer](https://github.com/xjewer)). +* Add MaterializedPostgreSQL table engine and database engine. Database engine allows to replicate a whole database or any subset of database tables. [#20470](https://github.com/ClickHouse/ClickHouse/pull/20470) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Adding support for a part of SQLJSON standard. [#24148](https://github.com/ClickHouse/ClickHouse/pull/24148) ([l1tsolaiki](https://github.com/l1tsolaiki)). +* Collect common system metrics (in `system.asynchronous_metrics` and `system.asynchronous_metric_log`) about CPU usage, disk usage, memory usage, IO, network, files, load average, CPU frequencies, thermal sensors, EDAC counters, system uptime; also added metrics about the scheduling jitter and the time spent collecting the metrics. It works like `atop` in ClickHouse and allows to get monitoring data even if you have no additional tools installed. This closes [#9430](https://github.com/ClickHouse/ClickHouse/issues/9430). [#24416](https://github.com/ClickHouse/ClickHouse/pull/24416) ([Yegor Levankov](https://github.com/elevankoff)). +* Add support `DISTINCT ON (columns)` expression, close [#25404](https://github.com/ClickHouse/ClickHouse/issues/25404). [#25589](https://github.com/ClickHouse/ClickHouse/pull/25589) ([Zijie Lu](https://github.com/TszKitLo40)). +* * Support Map type in `mapAdd` and `mapSubtract` functions * Support (U)Int128, U(Int256) types in `mapAdd` and `mapSubtract` functions. [#25596](https://github.com/ClickHouse/ClickHouse/pull/25596) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Add bin/unbin functions support. [#25609](https://github.com/ClickHouse/ClickHouse/pull/25609) ([zhaoyu](https://github.com/zxc111)). +* Introduce `system.data_skipping_indices` table containing information about existing data skipping indices. Closes [#7659](https://github.com/ClickHouse/ClickHouse/issues/7659). [#25693](https://github.com/ClickHouse/ClickHouse/pull/25693) ([Dmitry Novik](https://github.com/novikd)). +* in addition to https://github.com/ClickHouse/ClickHouse/pull/12073 add the FIRST keyword to the ADD INDEX command to be able to add index in the beginning of the indices list. [#25904](https://github.com/ClickHouse/ClickHouse/pull/25904) ([xjewer](https://github.com/xjewer)). +* Render pipelines as graphs in Web UI if `EXPLAIN PIPELINE graph = 1` query is given. [#26067](https://github.com/ClickHouse/ClickHouse/pull/26067) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add new functions `leftPad()`, `rightPad()`, `leftPadUTF8()`, `rightPadUTF8()`. [#26075](https://github.com/ClickHouse/ClickHouse/pull/26075) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Performance Improvement +* Added option to compile aggregate functions if `compile_aggregate_expressions` settings is on. [#24789](https://github.com/ClickHouse/ClickHouse/pull/24789) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Improvement +* `Database` argument for `StorageMerge` support regular expression. This closes #776. [#25064](https://github.com/ClickHouse/ClickHouse/pull/25064) ([flynn](https://github.com/ucasfl)). +* Allow extract non-string element as string using JSONExtract. This is for [#25414](https://github.com/ClickHouse/ClickHouse/issues/25414). [#25452](https://github.com/ClickHouse/ClickHouse/pull/25452) ([Amos Bird](https://github.com/amosbird)). +* Support for dynamic reloading of config to change number of threads in pool for background jobs execution (merges, mutations, fetches). [#25548](https://github.com/ClickHouse/ClickHouse/pull/25548) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Support TRUNCATE TABLE for StorageS3 and StorageHDFS. Closes [#25530](https://github.com/ClickHouse/ClickHouse/issues/25530). [#25550](https://github.com/ClickHouse/ClickHouse/pull/25550) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make `NetworkReceiveElapsedMicroseconds` metric to correctly include the time spent waiting for data from the client to INSERT. This closes [#9958](https://github.com/ClickHouse/ClickHouse/issues/9958). [#25602](https://github.com/ClickHouse/ClickHouse/pull/25602) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix possible logical race condition between `ALTER TABLE ... DETACH` and background merges. [#25605](https://github.com/ClickHouse/ClickHouse/pull/25605) ([Azat Khuzhin](https://github.com/azat)). +* Support materialized and aliased columns in joins, close [#13274](https://github.com/ClickHouse/ClickHouse/issues/13274). [#25634](https://github.com/ClickHouse/ClickHouse/pull/25634) ([Vladimir C](https://github.com/vdimir)). +* MaterializeMySQL now supports `ENUM` data type. [#25676](https://github.com/ClickHouse/ClickHouse/pull/25676) ([Storozhuk Kostiantyn](https://github.com/sand6255)). +* Cancel already running merges in partition on `DROP PARTITION` and `TRUNCATE` for `ReplicatedMergeTree`. Resolves [#17151](https://github.com/ClickHouse/ClickHouse/issues/17151). [#25684](https://github.com/ClickHouse/ClickHouse/pull/25684) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Use `Map` data type for key-value dictionaries in system logs tables (`system.query_log`, `system.query_thread_log`, `system.processes`, `system.opentelemetry_span_log`). Virtual columns are created to support old queries. Closes [#18698](https://github.com/ClickHouse/ClickHouse/issues/18698). Authors @hexiaoting, @sundy-li. [#25773](https://github.com/ClickHouse/ClickHouse/pull/25773) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix inconsistent behaviour of GROUP BY constant on empty set. Closes [#6842](https://github.com/ClickHouse/ClickHouse/issues/6842). [#25786](https://github.com/ClickHouse/ClickHouse/pull/25786) ([Kseniia Sumarokova](https://github.com/kssenii)). +* MySQL Engine now supports the exchange of column comments between MySQL and ClickHouse. [#25795](https://github.com/ClickHouse/ClickHouse/pull/25795) ([Storozhuk Kostiantyn](https://github.com/sand6255)). +* Fix "No available columns" for Merge() storage. [#25801](https://github.com/ClickHouse/ClickHouse/pull/25801) ([Azat Khuzhin](https://github.com/azat)). +* Allow to start clickhouse-client with unreadable working directory. [#25817](https://github.com/ClickHouse/ClickHouse/pull/25817) ([ianton-ru](https://github.com/ianton-ru)). +* Better handling of lost parts for ReplicatedMergeTree tables. Fixes rare inconsistencies in ReplicationQueue. Nothing should be visible to the user. Fixes [#10368](https://github.com/ClickHouse/ClickHouse/issues/10368). [#25820](https://github.com/ClickHouse/ClickHouse/pull/25820) ([alesapin](https://github.com/alesapin)). +* Fix an extremely rare bug which can lead to intersecting parts after `DROP PART` or background deletion of an empty part. [#25884](https://github.com/ClickHouse/ClickHouse/pull/25884) ([alesapin](https://github.com/alesapin)). +* Convert history file from readline format to replxx format. [#25888](https://github.com/ClickHouse/ClickHouse/pull/25888) ([Azat Khuzhin](https://github.com/azat)). +* Support LowCardinality, Decimal and UUID for JSON extract. Closes [#24606](https://github.com/ClickHouse/ClickHouse/issues/24606). [#25900](https://github.com/ClickHouse/ClickHouse/pull/25900) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add support for queries with a column named `"null"` (it must be specified in backticks or double quotes) and ON CLUSTER. This closes [#24035](https://github.com/ClickHouse/ClickHouse/issues/24035). [#25907](https://github.com/ClickHouse/ClickHouse/pull/25907) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Correctly throw exception on attempt to parse invalid Date. This closes [#6481](https://github.com/ClickHouse/ClickHouse/issues/6481). [#25909](https://github.com/ClickHouse/ClickHouse/pull/25909) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow parameters for parametric aggregate functions to be arbitrary constant expressions (e.g. `1 + 2`), not just literals. This also allows to use query parameters (in parametrized queries like `{param:UInt8}`) for parameters of parametric aggregate functions. This closes [#11607](https://github.com/ClickHouse/ClickHouse/issues/11607). [#25910](https://github.com/ClickHouse/ClickHouse/pull/25910) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow `quantiles*` functions to work with `aggregate_functions_null_for_empty`. This closes [#25892](https://github.com/ClickHouse/ClickHouse/issues/25892). [#25919](https://github.com/ClickHouse/ClickHouse/pull/25919) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `sudo service clickhouse-server start` to work on systems with `systemd` like Centos 8. This closes [#14298](https://github.com/ClickHouse/ClickHouse/issues/14298). This closes [#17799](https://github.com/ClickHouse/ClickHouse/issues/17799). [#25921](https://github.com/ClickHouse/ClickHouse/pull/25921) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add support for unicode (e.g. Chinese, Cyrillic) components in `Nested` data types. This closes [#25594](https://github.com/ClickHouse/ClickHouse/issues/25594). [#25923](https://github.com/ClickHouse/ClickHouse/pull/25923) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow complex quoted identifiers of JOINed tables. This closes [#17861](https://github.com/ClickHouse/ClickHouse/issues/17861). [#25924](https://github.com/ClickHouse/ClickHouse/pull/25924) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added setting `optimize_move_to_prewhere_if_final`. If query has `FINAL`, the optimization `move_to_prewhere` will be enabled only if both `optimize_move_to_prewhere` and `optimize_move_to_prewhere_if_final` are enabled. Closes [#8684](https://github.com/ClickHouse/ClickHouse/issues/8684). [#25940](https://github.com/ClickHouse/ClickHouse/pull/25940) ([Kseniia Sumarokova](https://github.com/kssenii)). +* More instrumentation for network interaction: add counters for recv/send bytes; add gauges for recvs/sends. Added missing documentation. This closes [#5897](https://github.com/ClickHouse/ClickHouse/issues/5897). [#25962](https://github.com/ClickHouse/ClickHouse/pull/25962) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Web UI: if value looks like an URL, automatically generate a link. [#25965](https://github.com/ClickHouse/ClickHouse/pull/25965) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error with query `SET SQL_SELECT_LIMIT` in mysql protocol. Closes [#17115](https://github.com/ClickHouse/ClickHouse/issues/17115). [#25972](https://github.com/ClickHouse/ClickHouse/pull/25972) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add support for argument of `UUID` type for `empty` and `notEmpty` functions. `UUID` is empty if it is all zeros (nil UUID). This closes [#3446](https://github.com/ClickHouse/ClickHouse/issues/3446). [#25974](https://github.com/ClickHouse/ClickHouse/pull/25974) ([zhaoyu](https://github.com/zxc111)). +* Add support for argument of AggregateFunction type for bin and hex functions. [#26094](https://github.com/ClickHouse/ClickHouse/pull/26094) ([zhaoyu](https://github.com/zxc111)). +* For dictionary with complex key if complex key contains only one attribute allow to not wrap key expression in tuple for functions `dictGet`, `dictHas`. [#26130](https://github.com/ClickHouse/ClickHouse/pull/26130) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Bug Fix +* `CAST` from `Date` to `DateTime` (or `DateTime64`) was not using the timezone of the `DateTime` type. It can also affect the comparison between `Date` and `DateTime`. Inference of the common type for `Date` and `DateTime` also was not using the corresponding timezone. It affected the results of function `if` and array construction. Closes [#24128](https://github.com/ClickHouse/ClickHouse/issues/24128). [#24129](https://github.com/ClickHouse/ClickHouse/pull/24129) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix assertion in PREWHERE with non-uint8 type, close [#19589](https://github.com/ClickHouse/ClickHouse/issues/19589). [#25484](https://github.com/ClickHouse/ClickHouse/pull/25484) ([Vladimir C](https://github.com/vdimir)). +* Fix `ALTER MODIFY COLUMN` of columns, which participates in TTL expressions. [#25554](https://github.com/ClickHouse/ClickHouse/pull/25554) ([Anton Popov](https://github.com/CurtizJ)). +* Fix slow dict join in some cases, close [#24209](https://github.com/ClickHouse/ClickHouse/issues/24209). [#25618](https://github.com/ClickHouse/ClickHouse/pull/25618) ([Vladimir C](https://github.com/vdimir)). +* Allow StorageMerge to access tables with aliases. Closes [#6051](https://github.com/ClickHouse/ClickHouse/issues/6051). [#25694](https://github.com/ClickHouse/ClickHouse/pull/25694) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix bug in `TTL` with `GROUP BY` expression which refuses to execute `TTL` after first execution in part. [#25743](https://github.com/ClickHouse/ClickHouse/pull/25743) ([alesapin](https://github.com/alesapin)). +* Fix rare bug with `DROP PART` query for `ReplicatedMergeTree` tables which can lead to error message `Unexpected merged part intersecting drop range`. [#25783](https://github.com/ClickHouse/ClickHouse/pull/25783) ([alesapin](https://github.com/alesapin)). +* Fix ARM exception handling with non default page size. Fixes [#25512](https://github.com/ClickHouse/ClickHouse/issues/25512). Fixes [#25044](https://github.com/ClickHouse/ClickHouse/issues/25044). Fixes [#24901](https://github.com/ClickHouse/ClickHouse/issues/24901). Fixes [#23183](https://github.com/ClickHouse/ClickHouse/issues/23183). Fixes [#20221](https://github.com/ClickHouse/ClickHouse/issues/20221). Fixes [#19703](https://github.com/ClickHouse/ClickHouse/issues/19703). Fixes [#19028](https://github.com/ClickHouse/ClickHouse/issues/19028). Fixes [#18391](https://github.com/ClickHouse/ClickHouse/issues/18391). Fixes [#18121](https://github.com/ClickHouse/ClickHouse/issues/18121). Fixes [#17994](https://github.com/ClickHouse/ClickHouse/issues/17994). Fixes [#12483](https://github.com/ClickHouse/ClickHouse/issues/12483). [#25854](https://github.com/ClickHouse/ClickHouse/pull/25854) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). +* Fixed `scram-sha-256` authentication for PostgreSQL engines. Closes [#24516](https://github.com/ClickHouse/ClickHouse/issues/24516). [#25906](https://github.com/ClickHouse/ClickHouse/pull/25906) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix crash on call dictGet() with bad arguments. [#25913](https://github.com/ClickHouse/ClickHouse/pull/25913) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix possible deadlock during query profiler stack unwinding. Fixes [#25968](https://github.com/ClickHouse/ClickHouse/issues/25968). [#25970](https://github.com/ClickHouse/ClickHouse/pull/25970) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix formatting of type `Map` with integer keys to `JSON`. [#25982](https://github.com/ClickHouse/ClickHouse/pull/25982) ([Anton Popov](https://github.com/CurtizJ)). +* Fix wrong thread estimation for right subquery join in some cases. Close [#24075](https://github.com/ClickHouse/ClickHouse/issues/24075). [#26052](https://github.com/ClickHouse/ClickHouse/pull/26052) ([Vladimir C](https://github.com/vdimir)). +* Fix rare server crash because of `abort` in ZooKeeper client. Fixes [#25813](https://github.com/ClickHouse/ClickHouse/issues/25813). [#26079](https://github.com/ClickHouse/ClickHouse/pull/26079) ([alesapin](https://github.com/alesapin)). +* Fix throwing exception when iterate over non existing remote directory. [#26087](https://github.com/ClickHouse/ClickHouse/pull/26087) ([ianton-ru](https://github.com/ianton-ru)). +* Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `joinGet` with LowCarinality columns, close [#25993](https://github.com/ClickHouse/ClickHouse/issues/25993). [#26118](https://github.com/ClickHouse/ClickHouse/pull/26118) ([Vladimir C](https://github.com/vdimir)). + +#### Build/Testing/Packaging Improvement +* - Syntax update: changing 'is' to '==' when necessary. [#25559](https://github.com/ClickHouse/ClickHouse/pull/25559) ([MyroTk](https://github.com/MyroTk)). +* Add new tests for checking access rights for columns used in filters (WHERE / PREWHERE / row policy) of the `SELECT` statement after changes in https://github.com/ClickHouse/ClickHouse/pull/24405. [#25619](https://github.com/ClickHouse/ClickHouse/pull/25619) ([Vitaly Baranov](https://github.com/vitlibar)). +* Enabling all TestFlows modules and fixing some tests. [#26011](https://github.com/ClickHouse/ClickHouse/pull/26011) ([vzakaznikov](https://github.com/vzakaznikov)). +* Disabling TestFlows LDAP module due to test fails. [#26065](https://github.com/ClickHouse/ClickHouse/pull/26065) ([vzakaznikov](https://github.com/vzakaznikov)). + +#### Other +* Add `clickhouse-keeper-converter` tool which allows converting zookeeper logs and snapshots into `clickhouse-keeper` snapshot format. [#25428](https://github.com/ClickHouse/ClickHouse/pull/25428) ([alesapin](https://github.com/alesapin)). + +#### NO CL ENTRY + +* NO CL ENTRY: '[ImgBot] Optimize images'. [#26054](https://github.com/ClickHouse/ClickHouse/pull/26054) ([imgbot[bot]](https://github.com/apps/imgbot)). + diff --git a/docs/changelogs/v21.8.10.19-lts.md b/docs/changelogs/v21.8.10.19-lts.md new file mode 100644 index 00000000000..92347642ffa --- /dev/null +++ b/docs/changelogs/v21.8.10.19-lts.md @@ -0,0 +1,12 @@ +### ClickHouse release v21.8.10.19-lts FIXME as compared to v21.8.9.13-lts + +#### Improvement +* Backported in [#30452](https://github.com/ClickHouse/ClickHouse/issues/30452): Allow symlinks to files in user_files directory for file table function. [#30309](https://github.com/ClickHouse/ClickHouse/pull/30309) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#29724](https://github.com/ClickHouse/ClickHouse/issues/29724): Fix null deference for `GROUP BY WITH TOTALS HAVING` (when the column from `HAVING` wasn't selected). [#29553](https://github.com/ClickHouse/ClickHouse/pull/29553) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30233](https://github.com/ClickHouse/ClickHouse/issues/30233): Fix INSERT SELECT incorrectly fills MATERIALIZED column based of Nullable column. [#30189](https://github.com/ClickHouse/ClickHouse/pull/30189) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30333](https://github.com/ClickHouse/ClickHouse/issues/30333): * Allow identifiers staring with numbers in multiple joins. [#30230](https://github.com/ClickHouse/ClickHouse/pull/30230) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#30377](https://github.com/ClickHouse/ClickHouse/issues/30377): fix replaceRegexpAll bug. [#30292](https://github.com/ClickHouse/ClickHouse/pull/30292) ([Memo](https://github.com/Joeywzr)). + diff --git a/docs/changelogs/v21.8.11.4-lts.md b/docs/changelogs/v21.8.11.4-lts.md new file mode 100644 index 00000000000..e36d75c32ea --- /dev/null +++ b/docs/changelogs/v21.8.11.4-lts.md @@ -0,0 +1,40 @@ +### ClickHouse release v21.8.11.4-lts FIXME as compared to v21.8.10.19-lts + +#### New Feature +* Backported in [#30713](https://github.com/ClickHouse/ClickHouse/issues/30713): CompiledExpressionCache limit elements size using `compiled_expression_cache_elements_size` setting. [#30667](https://github.com/ClickHouse/ClickHouse/pull/30667) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Improvement +* Backported in [#30621](https://github.com/ClickHouse/ClickHouse/issues/30621): Make query, which fetched table structure for PostgreSQL database because, more reliable. [#30477](https://github.com/ClickHouse/ClickHouse/pull/30477) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Bug Fix +* Backported in [#31368](https://github.com/ClickHouse/ClickHouse/issues/31368): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release + +* Backported in [#30914](https://github.com/ClickHouse/ClickHouse/issues/30914): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#28756](https://github.com/ClickHouse/ClickHouse/issues/28756): Fix NOT-IN index optimization when not all key columns are used. This fixes [#28120](https://github.com/ClickHouse/ClickHouse/issues/28120). [#28315](https://github.com/ClickHouse/ClickHouse/pull/28315) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#30825](https://github.com/ClickHouse/ClickHouse/issues/30825): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29175](https://github.com/ClickHouse/ClickHouse/issues/29175): Fix queries to external databases (i.e. MySQL) with multiple columns in IN ( i.e. `(k,v) IN ((1, 2))` ) (but note that this has some backward incompatibility for the `clickhouse-copier` since it uses alias for tuple element). [#28888](https://github.com/ClickHouse/ClickHouse/pull/28888) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30610](https://github.com/ClickHouse/ClickHouse/issues/30610): Fix bad optimizations of ORDER BY if it contains WITH FILL. This closes [#28908](https://github.com/ClickHouse/ClickHouse/issues/28908). This closes [#26049](https://github.com/ClickHouse/ClickHouse/issues/26049). [#28910](https://github.com/ClickHouse/ClickHouse/pull/28910) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#30767](https://github.com/ClickHouse/ClickHouse/issues/30767): Fix hanging DDL queries on Replicated database while adding a new replica. [#29328](https://github.com/ClickHouse/ClickHouse/pull/29328) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Backported in [#30205](https://github.com/ClickHouse/ClickHouse/issues/30205): Fix data-race between `LogSink::writeMarks()` and `LogSource` in `StorageLog`. [#29946](https://github.com/ClickHouse/ClickHouse/pull/29946) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30463](https://github.com/ClickHouse/ClickHouse/issues/30463): Support nullable arguments in function `initializeAggregation`. [#30177](https://github.com/ClickHouse/ClickHouse/pull/30177) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#30655](https://github.com/ClickHouse/ClickHouse/issues/30655): Fix `[I]LIKE` function. Closes [#28661](https://github.com/ClickHouse/ClickHouse/issues/28661). [#30244](https://github.com/ClickHouse/ClickHouse/pull/30244) ([Nikolay Degterinsky](https://github.com/evillique)). +* Backported in [#30289](https://github.com/ClickHouse/ClickHouse/issues/30289): Fix ComplexKeyHashedDictionary, ComplexKeySparseHashedDictionary parsing `preallocate` option from layout config. [#30246](https://github.com/ClickHouse/ClickHouse/pull/30246) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#30525](https://github.com/ClickHouse/ClickHouse/issues/30525): Fixed segfault which might happen if session expired during execution of REPLACE PARTITION. [#30432](https://github.com/ClickHouse/ClickHouse/pull/30432) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix exception handling in `parallel_view_processing`. That resolves issues / prevents crashes in some rare corner cases, when that feature is enabled and exception (like "Memory limit exceeded ...") happened in the middle of materialized view processing. [#30472](https://github.com/ClickHouse/ClickHouse/pull/30472) ([filimonov](https://github.com/filimonov)). +* Backported in [#30585](https://github.com/ClickHouse/ClickHouse/issues/30585): * Fix deadlock on ALTER with scalar subquery to the same table, close [#30461](https://github.com/ClickHouse/ClickHouse/issues/30461). [#30492](https://github.com/ClickHouse/ClickHouse/pull/30492) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#30607](https://github.com/ClickHouse/ClickHouse/issues/30607): Limit push down optimization could cause a error `Cannot find column`. Fixes [#30438](https://github.com/ClickHouse/ClickHouse/issues/30438). [#30562](https://github.com/ClickHouse/ClickHouse/pull/30562) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#30753](https://github.com/ClickHouse/ClickHouse/issues/30753): Functions for case-insensitive search in UTF8 strings like `positionCaseInsensitiveUTF8` and `countSubstringsCaseInsensitiveUTF8` might find substrings that actually does not match, it's fixed. [#30663](https://github.com/ClickHouse/ClickHouse/pull/30663) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30710](https://github.com/ClickHouse/ClickHouse/issues/30710): Fix PREWHERE with WHERE in case of always true PREWHERE. [#30668](https://github.com/ClickHouse/ClickHouse/pull/30668) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30770](https://github.com/ClickHouse/ClickHouse/issues/30770): Fixed a race condition between `REPLACE/MOVE PARTITION` and background merge in non-replicated `MergeTree` that might cause a part of moved/replaced data to remain in partition. Fixes [#29327](https://github.com/ClickHouse/ClickHouse/issues/29327). [#30717](https://github.com/ClickHouse/ClickHouse/pull/30717) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30857](https://github.com/ClickHouse/ClickHouse/issues/30857): Fixed ambiguity when extracting auxiliary ZooKeeper name from ZooKeeper path in `ReplicatedMergeTree`. Previously server might fail to start with `Unknown auxiliary ZooKeeper name` if ZooKeeper path contains a colon. Fixes [#29052](https://github.com/ClickHouse/ClickHouse/issues/29052). Also it was allowed to specify ZooKeeper path that does not start with slash, but now it's deprecated and creation of new tables with such path is not allowed. Slashes and colons in auxiliary ZooKeeper names are not allowed too. [#30822](https://github.com/ClickHouse/ClickHouse/pull/30822) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30925](https://github.com/ClickHouse/ClickHouse/issues/30925): Fix set index not used in AND/OR expressions when there are more than two operands. This fixes [#30416](https://github.com/ClickHouse/ClickHouse/issues/30416) . [#30887](https://github.com/ClickHouse/ClickHouse/pull/30887) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#31151](https://github.com/ClickHouse/ClickHouse/issues/31151): Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)). +* Backported in [#31040](https://github.com/ClickHouse/ClickHouse/issues/31040): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31132](https://github.com/ClickHouse/ClickHouse/issues/31132): Fix JSONValue/Query with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31372](https://github.com/ClickHouse/ClickHouse/issues/31372): Fix StorageMerge with aliases and where (it did not work before at all). Closes [#28802](https://github.com/ClickHouse/ClickHouse/issues/28802). [#31044](https://github.com/ClickHouse/ClickHouse/pull/31044) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.8.12.29-lts.md b/docs/changelogs/v21.8.12.29-lts.md new file mode 100644 index 00000000000..63b34c367f3 --- /dev/null +++ b/docs/changelogs/v21.8.12.29-lts.md @@ -0,0 +1,24 @@ +### ClickHouse release v21.8.12.29-lts FIXME as compared to v21.8.11.4-lts + +#### Performance Improvement +* Backported in [#31732](https://github.com/ClickHouse/ClickHouse/issues/31732): Improve performance of JSON and XML output formats. [#31673](https://github.com/ClickHouse/ClickHouse/pull/31673) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#31575](https://github.com/ClickHouse/ClickHouse/issues/31575): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#31204](https://github.com/ClickHouse/ClickHouse/issues/31204): Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31253](https://github.com/ClickHouse/ClickHouse/issues/31253): Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)). +* Backported in [#31521](https://github.com/ClickHouse/ClickHouse/issues/31521): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)). +* Backported in [#31552](https://github.com/ClickHouse/ClickHouse/issues/31552): Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#31582](https://github.com/ClickHouse/ClickHouse/issues/31582): * Disable `partial_merge_join_left_table_buffer_bytes` before bug in this optimization is fixed. See [#31009](https://github.com/ClickHouse/ClickHouse/issues/31009)). * Remove redundant option `partial_merge_join_optimizations`. [#31528](https://github.com/ClickHouse/ClickHouse/pull/31528) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#31599](https://github.com/ClickHouse/ClickHouse/issues/31599): Fix invalid generated JSON when only column names contain invalid UTF-8 sequences. [#31534](https://github.com/ClickHouse/ClickHouse/pull/31534) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Backported in [#31779](https://github.com/ClickHouse/ClickHouse/issues/31779): `RENAME TABLE` query worked incorrectly on attempt to rename an DDL dictionary in `Ordinary` database, it's fixed. [#31638](https://github.com/ClickHouse/ClickHouse/pull/31638) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31790](https://github.com/ClickHouse/ClickHouse/issues/31790): Settings `input_format_allow_errors_num` and `input_format_allow_errors_ratio` did not work for parsing of domain types, such as `IPv4`, it's fixed. Fixes [#31686](https://github.com/ClickHouse/ClickHouse/issues/31686). [#31697](https://github.com/ClickHouse/ClickHouse/pull/31697) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31829](https://github.com/ClickHouse/ClickHouse/issues/31829): Fixed `there are no such cluster here` error on execution of `ON CLUSTER` query if specified cluster name is name of `Replicated` database. [#31723](https://github.com/ClickHouse/ClickHouse/pull/31723) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31813](https://github.com/ClickHouse/ClickHouse/issues/31813): Fix race in JSONEachRowWithProgress output format when data and lines with progress are mixed in output. [#31736](https://github.com/ClickHouse/ClickHouse/pull/31736) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31761](https://github.com/ClickHouse/ClickHouse/issues/31761): Fix usage of `Buffer` table engine with type `Map`. Fixes [#30546](https://github.com/ClickHouse/ClickHouse/issues/30546). [#31742](https://github.com/ClickHouse/ClickHouse/pull/31742) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#31890](https://github.com/ClickHouse/ClickHouse/issues/31890): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31910](https://github.com/ClickHouse/ClickHouse/issues/31910): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.8.13.6-lts.md b/docs/changelogs/v21.8.13.6-lts.md new file mode 100644 index 00000000000..06e8c366ff2 --- /dev/null +++ b/docs/changelogs/v21.8.13.6-lts.md @@ -0,0 +1,29 @@ +### ClickHouse release v21.8.13.6-lts FIXME as compared to v21.8.12.29-lts + +#### Bug Fix +* Backported in [#32688](https://github.com/ClickHouse/ClickHouse/issues/32688): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). +* Backported in [#32343](https://github.com/ClickHouse/ClickHouse/issues/32343): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#32108](https://github.com/ClickHouse/ClickHouse/issues/32108): Fix crash with empty result on odbc query. Closes [#31465](https://github.com/ClickHouse/ClickHouse/issues/31465). [#31766](https://github.com/ClickHouse/ClickHouse/pull/31766) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#32150](https://github.com/ClickHouse/ClickHouse/issues/32150): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#32075](https://github.com/ClickHouse/ClickHouse/issues/32075): Fix a bug about function transform with decimal args. [#31839](https://github.com/ClickHouse/ClickHouse/pull/31839) ([Shuai li](https://github.com/loneylee)). +* Backported in [#31955](https://github.com/ClickHouse/ClickHouse/issues/31955): - Change configuration path from `keeper_server.session_timeout_ms` to `keeper_server.coordination_settings.session_timeout_ms` when constructing a `KeeperTCPHandler` - Same with `operation_timeout`. [#31859](https://github.com/ClickHouse/ClickHouse/pull/31859) ([JackyWoo](https://github.com/JackyWoo)). +* Backported in [#32161](https://github.com/ClickHouse/ClickHouse/issues/32161): Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32146](https://github.com/ClickHouse/ClickHouse/issues/32146): Fixed `Directory ... already exists and is not empty` error when detaching part. [#32063](https://github.com/ClickHouse/ClickHouse/pull/32063) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32213](https://github.com/ClickHouse/ClickHouse/issues/32213): Number of active replicas might be determined incorrectly when inserting with quorum if setting `replicated_can_become_leader` is disabled on some replicas. It's fixed. [#32157](https://github.com/ClickHouse/ClickHouse/pull/32157) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32312](https://github.com/ClickHouse/ClickHouse/issues/32312): XML dictionaries identifiers, used in table create query, can be qualified to `default_database` during upgrade to newer version. Closes [#31963](https://github.com/ClickHouse/ClickHouse/issues/31963). [#32187](https://github.com/ClickHouse/ClickHouse/pull/32187) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#32540](https://github.com/ClickHouse/ClickHouse/issues/32540): Some replication queue entries might hang for `temporary_directories_lifetime` (1 day by default) with `Directory tmp_merge_` or `Part ... (state Deleting) already exists, but it will be deleted soon` or similar error. It's fixed. Fixes [#29616](https://github.com/ClickHouse/ClickHouse/issues/29616). [#32201](https://github.com/ClickHouse/ClickHouse/pull/32201) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#32353](https://github.com/ClickHouse/ClickHouse/issues/32353): Fixed crash with SIGFPE in aggregate function `avgWeighted` with `Decimal` argument. Fixes [#32053](https://github.com/ClickHouse/ClickHouse/issues/32053). [#32303](https://github.com/ClickHouse/ClickHouse/pull/32303) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#33048](https://github.com/ClickHouse/ClickHouse/issues/33048): Fix possible exception at RabbitMQ storage startup by delaying channel creation. [#32584](https://github.com/ClickHouse/ClickHouse/pull/32584) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#32795](https://github.com/ClickHouse/ClickHouse/issues/32795): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release): + +* Backported in [#32659](https://github.com/ClickHouse/ClickHouse/issues/32659): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'fix json error after downgrade'. [#33166](https://github.com/ClickHouse/ClickHouse/pull/33166) ([bullet1337](https://github.com/bullet1337)). + diff --git a/docs/changelogs/v21.8.14.5-lts.md b/docs/changelogs/v21.8.14.5-lts.md new file mode 100644 index 00000000000..481327a35c9 --- /dev/null +++ b/docs/changelogs/v21.8.14.5-lts.md @@ -0,0 +1,8 @@ +### ClickHouse release v21.8.14.5-lts FIXME as compared to v21.8.13.6-lts + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#33184](https://github.com/ClickHouse/ClickHouse/issues/33184): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#33659](https://github.com/ClickHouse/ClickHouse/issues/33659): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#33206](https://github.com/ClickHouse/ClickHouse/issues/33206): Fix s3 table function reading empty file. Closes [#33008](https://github.com/ClickHouse/ClickHouse/issues/33008). [#33037](https://github.com/ClickHouse/ClickHouse/pull/33037) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.8.15.7-lts.md b/docs/changelogs/v21.8.15.7-lts.md new file mode 100644 index 00000000000..7411fbff9ae --- /dev/null +++ b/docs/changelogs/v21.8.15.7-lts.md @@ -0,0 +1,7 @@ +### ClickHouse release v21.8.15.7-lts FIXME as compared to v21.8.14.5-lts + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#34121](https://github.com/ClickHouse/ClickHouse/issues/34121): Fix usage of functions `array` and `tuple` with literal arguments in distributed queries. Previously it could lead to `Not found columns` exception. [#33938](https://github.com/ClickHouse/ClickHouse/pull/33938) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#34097](https://github.com/ClickHouse/ClickHouse/issues/34097): Fix segfault while parsing ORC file with corrupted footer. Closes [#33797](https://github.com/ClickHouse/ClickHouse/issues/33797). [#33984](https://github.com/ClickHouse/ClickHouse/pull/33984) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.8.2.19-prestable.md b/docs/changelogs/v21.8.2.19-prestable.md new file mode 100644 index 00000000000..15726747e65 --- /dev/null +++ b/docs/changelogs/v21.8.2.19-prestable.md @@ -0,0 +1,29 @@ +### ClickHouse release v21.8.2.19-prestable FIXME as compared to v21.8.1.7409-prestable + +#### Performance Improvement +* Backported in [#26524](https://github.com/ClickHouse/ClickHouse/issues/26524): Improve latency of short queries, that require reading from tables with large number of columns. [#26371](https://github.com/ClickHouse/ClickHouse/pull/26371) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix +* Backported in [#26283](https://github.com/ClickHouse/ClickHouse/issues/26283): Fix `optimize_skip_unused_shards_rewrite_in` for non-UInt64 types (may select incorrect shards eventually or throw `Cannot infer type of an empty tuple` or `Function tuple requires at least one argument`). [#25798](https://github.com/ClickHouse/ClickHouse/pull/25798) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#26587](https://github.com/ClickHouse/ClickHouse/issues/26587): Fixed `Not found column ...` and `Missing column ...` errors when selecting from `MaterializeMySQL`. Fixes [#23708](https://github.com/ClickHouse/ClickHouse/issues/23708), [#24830](https://github.com/ClickHouse/ClickHouse/issues/24830), [#25794](https://github.com/ClickHouse/ClickHouse/issues/25794). [#25822](https://github.com/ClickHouse/ClickHouse/pull/25822) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#26190](https://github.com/ClickHouse/ClickHouse/issues/26190): Fix sharding_key from column w/o function for remote() (before `select * from remote('127.1', system.one, dummy)` leads to `Unknown column: dummy, there are only columns .` error). [#25824](https://github.com/ClickHouse/ClickHouse/pull/25824) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#26284](https://github.com/ClickHouse/ClickHouse/issues/26284): Fix possible mismatched header when using normal projection with prewhere. This fixes [#26020](https://github.com/ClickHouse/ClickHouse/issues/26020). [#26038](https://github.com/ClickHouse/ClickHouse/pull/26038) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#26298](https://github.com/ClickHouse/ClickHouse/issues/26298): Fixed incorrect `sequence_id` in MySQL protocol packets that ClickHouse sends on exception during query execution. It might cause MySQL client to reset connection to ClickHouse server. Fixes [#21184](https://github.com/ClickHouse/ClickHouse/issues/21184). [#26051](https://github.com/ClickHouse/ClickHouse/pull/26051) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#26204](https://github.com/ClickHouse/ClickHouse/issues/26204): Fix potential crash if more than one `untuple` expression is used. [#26179](https://github.com/ClickHouse/ClickHouse/pull/26179) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#26228](https://github.com/ClickHouse/ClickHouse/issues/26228): Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#26358](https://github.com/ClickHouse/ClickHouse/issues/26358): Fix logical error on join with totals, close [#26017](https://github.com/ClickHouse/ClickHouse/issues/26017). [#26250](https://github.com/ClickHouse/ClickHouse/pull/26250) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#26788](https://github.com/ClickHouse/ClickHouse/issues/26788): Fix zstd decompression in case there are escape sequences at the end of internal buffer. Closes [#26013](https://github.com/ClickHouse/ClickHouse/issues/26013). [#26314](https://github.com/ClickHouse/ClickHouse/pull/26314) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#26360](https://github.com/ClickHouse/ClickHouse/issues/26360): Fixed rare bug in lost replica recovery that may cause replicas to diverge. [#26321](https://github.com/ClickHouse/ClickHouse/pull/26321) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#26452](https://github.com/ClickHouse/ClickHouse/issues/26452): Fix `optimize_distributed_group_by_sharding_key` for multiple columns (leads to incorrect result w/ `optimize_skip_unused_shards=1`/`allow_nondeterministic_optimize_skip_unused_shards=1` and multiple columns in sharding key expression). [#26353](https://github.com/ClickHouse/ClickHouse/pull/26353) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#26422](https://github.com/ClickHouse/ClickHouse/issues/26422): Fix possible crash when login as dropped user. This PR fixes [#26073](https://github.com/ClickHouse/ClickHouse/issues/26073). [#26363](https://github.com/ClickHouse/ClickHouse/pull/26363) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#26417](https://github.com/ClickHouse/ClickHouse/issues/26417): Fix infinite non joined block stream in `partial_merge_join` close [#26325](https://github.com/ClickHouse/ClickHouse/issues/26325). [#26374](https://github.com/ClickHouse/ClickHouse/pull/26374) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#26448](https://github.com/ClickHouse/ClickHouse/issues/26448): Fix some fuzzed msan crash. Fixes [#22517](https://github.com/ClickHouse/ClickHouse/issues/22517). [#26428](https://github.com/ClickHouse/ClickHouse/pull/26428) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#26488](https://github.com/ClickHouse/ClickHouse/issues/26488): Fix broken name resolution after rewriting column aliases. This fixes [#26432](https://github.com/ClickHouse/ClickHouse/issues/26432). [#26475](https://github.com/ClickHouse/ClickHouse/pull/26475) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#26613](https://github.com/ClickHouse/ClickHouse/issues/26613): Fix issues with `CREATE DICTIONARY` query if dictionary name or database name was quoted. Closes [#26491](https://github.com/ClickHouse/ClickHouse/issues/26491). [#26508](https://github.com/ClickHouse/ClickHouse/pull/26508) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#26605](https://github.com/ClickHouse/ClickHouse/issues/26605): Fix crash in rabbitmq shutdown in case rabbitmq setup was not started. Closes [#26504](https://github.com/ClickHouse/ClickHouse/issues/26504). [#26529](https://github.com/ClickHouse/ClickHouse/pull/26529) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#26608](https://github.com/ClickHouse/ClickHouse/issues/26608): Update `chown` cmd check in clickhouse-server docker entrypoint. It fixes the bug that cluster pod restart failed (or timeout) on kubernetes. [#26545](https://github.com/ClickHouse/ClickHouse/pull/26545) ([Ky Li](https://github.com/Kylinrix)). +* Backported in [#26648](https://github.com/ClickHouse/ClickHouse/issues/26648): Fix incorrect function names of groupBitmapAnd/Or/Xor. This fixes. [#26557](https://github.com/ClickHouse/ClickHouse/pull/26557) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#26644](https://github.com/ClickHouse/ClickHouse/issues/26644): Fix history file conversion if file is empty. [#26589](https://github.com/ClickHouse/ClickHouse/pull/26589) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#26705](https://github.com/ClickHouse/ClickHouse/issues/26705): Fix potential nullptr dereference in window functions. This fixes [#25276](https://github.com/ClickHouse/ClickHouse/issues/25276). [#26668](https://github.com/ClickHouse/ClickHouse/pull/26668) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Backported in [#26771](https://github.com/ClickHouse/ClickHouse/issues/26771): Sometimes SET ROLE could work incorrectly, this PR fixes that. [#26707](https://github.com/ClickHouse/ClickHouse/pull/26707) ([Vitaly Baranov](https://github.com/vitlibar)). + diff --git a/docs/changelogs/v21.8.3.44-lts.md b/docs/changelogs/v21.8.3.44-lts.md new file mode 100644 index 00000000000..21fe655870a --- /dev/null +++ b/docs/changelogs/v21.8.3.44-lts.md @@ -0,0 +1,28 @@ +### ClickHouse release v21.8.3.44-lts FIXME as compared to v21.8.2.19-prestable + +#### Improvement +* Rename setting. [#26844](https://github.com/ClickHouse/ClickHouse/pull/26844) ([ianton-ru](https://github.com/ianton-ru)). + +#### Bug Fix +* Backported in [#27545](https://github.com/ClickHouse/ClickHouse/issues/27545): Now, scalar subquery always returns `Nullable` result if it's type can be `Nullable`. It is needed because in case of empty subquery it's result should be `Null`. Previously, it was possible to get error about incompatible types (type deduction does not execute scalar subquery, and it could use not-nullable type). Scalar subquery with empty result which can't be converted to `Nullable` (like `Array` or `Tuple`) now throws error. Fixes [#25411](https://github.com/ClickHouse/ClickHouse/issues/25411). [#26423](https://github.com/ClickHouse/ClickHouse/pull/26423) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#26856](https://github.com/ClickHouse/ClickHouse/issues/26856): ParallelFormattingOutputFormat: Use mutex to handle the join to the collector_thread (https://github.com/ClickHouse/ClickHouse/issues/26694). [#26703](https://github.com/ClickHouse/ClickHouse/pull/26703) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#26941](https://github.com/ClickHouse/ClickHouse/issues/26941): Do not remove data on ReplicatedMergeTree table shutdown to avoid creating data to metadata inconsistency. [#26716](https://github.com/ClickHouse/ClickHouse/pull/26716) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#26981](https://github.com/ClickHouse/ClickHouse/issues/26981): Aggregate function parameters might be lost when applying some combinators causing exceptions like `Conversion from AggregateFunction(topKArray, Array(String)) to AggregateFunction(topKArray(10), Array(String)) is not supported`. It's fixed. Fixes [#26196](https://github.com/ClickHouse/ClickHouse/issues/26196) and [#26433](https://github.com/ClickHouse/ClickHouse/issues/26433). [#26814](https://github.com/ClickHouse/ClickHouse/pull/26814) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#26909](https://github.com/ClickHouse/ClickHouse/issues/26909): Fix library-bridge ids load. [#26834](https://github.com/ClickHouse/ClickHouse/pull/26834) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#26945](https://github.com/ClickHouse/ClickHouse/issues/26945): Fix error `Missing columns: 'xxx'` when `DEFAULT` column references other non materialized column without `DEFAULT` expression. Fixes [#26591](https://github.com/ClickHouse/ClickHouse/issues/26591). [#26900](https://github.com/ClickHouse/ClickHouse/pull/26900) ([alesapin](https://github.com/alesapin)). +* Backported in [#26997](https://github.com/ClickHouse/ClickHouse/issues/26997): Fix reading of custom TLDs (stops processing with lower buffer or bigger file). [#26948](https://github.com/ClickHouse/ClickHouse/pull/26948) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27030](https://github.com/ClickHouse/ClickHouse/issues/27030): Now partition ID in queries like `ALTER TABLE ... PARTITION ID xxx` validates for correctness. Fixes [#25718](https://github.com/ClickHouse/ClickHouse/issues/25718). [#26963](https://github.com/ClickHouse/ClickHouse/pull/26963) ([alesapin](https://github.com/alesapin)). +* Backported in [#27053](https://github.com/ClickHouse/ClickHouse/issues/27053): [RFC] Fix possible mutation stack due to race with DROP_RANGE. [#27002](https://github.com/ClickHouse/ClickHouse/pull/27002) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27106](https://github.com/ClickHouse/ClickHouse/issues/27106): Fixed `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache` configuration parsing. Options `allow_read_expired_keys`, `max_update_queue_size`, `update_queue_push_timeout_milliseconds`, `query_wait_timeout_milliseconds` were not parsed for dictionaries with non `cache` type. [#27032](https://github.com/ClickHouse/ClickHouse/pull/27032) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#27160](https://github.com/ClickHouse/ClickHouse/issues/27160): Fix synchronization in GRPCServer This PR fixes [#27024](https://github.com/ClickHouse/ClickHouse/issues/27024). [#27064](https://github.com/ClickHouse/ClickHouse/pull/27064) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#27365](https://github.com/ClickHouse/ClickHouse/issues/27365): - Fix uninitialized memory in functions `multiSearch*` with empty array, close [#27169](https://github.com/ClickHouse/ClickHouse/issues/27169). [#27181](https://github.com/ClickHouse/ClickHouse/pull/27181) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#27263](https://github.com/ClickHouse/ClickHouse/issues/27263): In rare cases `system.detached_parts` table might contain incorrect information for some parts, it's fixed. Fixes [#27114](https://github.com/ClickHouse/ClickHouse/issues/27114). [#27183](https://github.com/ClickHouse/ClickHouse/pull/27183) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#27555](https://github.com/ClickHouse/ClickHouse/issues/27555): Fix mutation stuck on invalid partitions in non-replicated MergeTree. [#27248](https://github.com/ClickHouse/ClickHouse/pull/27248) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27411](https://github.com/ClickHouse/ClickHouse/issues/27411): Fix `distributed_group_by_no_merge=2`+`distributed_push_down_limit=1` or `optimize_distributed_group_by_sharding_key=1` with `LIMIT BY` and `LIMIT OFFSET`. [#27249](https://github.com/ClickHouse/ClickHouse/pull/27249) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#27507](https://github.com/ClickHouse/ClickHouse/issues/27507): Fix errors like `Expected ColumnLowCardinality, gotUInt8` or `Bad cast from type DB::ColumnVector to DB::ColumnLowCardinality` for some queries with `LowCardinality` in `PREWHERE`. Fixes [#23515](https://github.com/ClickHouse/ClickHouse/issues/23515). [#27298](https://github.com/ClickHouse/ClickHouse/pull/27298) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#27417](https://github.com/ClickHouse/ClickHouse/issues/27417): Fix `Cannot find column` error for queries with sampling. Was introduced in [#24574](https://github.com/ClickHouse/ClickHouse/issues/24574). Fixes [#26522](https://github.com/ClickHouse/ClickHouse/issues/26522). [#27301](https://github.com/ClickHouse/ClickHouse/pull/27301) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#27414](https://github.com/ClickHouse/ClickHouse/issues/27414): Fixed incorrect validation of partition id for MergeTree tables that created with old syntax. [#27328](https://github.com/ClickHouse/ClickHouse/pull/27328) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#27540](https://github.com/ClickHouse/ClickHouse/issues/27540): Fix incorrect result for query with row-level security, prewhere and LowCardinality filter. Fixes [#27179](https://github.com/ClickHouse/ClickHouse/issues/27179). [#27329](https://github.com/ClickHouse/ClickHouse/pull/27329) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#27419](https://github.com/ClickHouse/ClickHouse/issues/27419): /proc/info contains metrics like. [#27361](https://github.com/ClickHouse/ClickHouse/pull/27361) ([Mike Kot](https://github.com/myrrc)). +* Backported in [#27472](https://github.com/ClickHouse/ClickHouse/issues/27472): fix metric BackgroundMessageBrokerSchedulePoolTask, maybe mistyped。. [#27452](https://github.com/ClickHouse/ClickHouse/pull/27452) ([Ben](https://github.com/benbiti)). + diff --git a/docs/changelogs/v21.8.4.51-lts.md b/docs/changelogs/v21.8.4.51-lts.md new file mode 100644 index 00000000000..a8494ebb1d1 --- /dev/null +++ b/docs/changelogs/v21.8.4.51-lts.md @@ -0,0 +1,11 @@ +### ClickHouse release v21.8.4.51-lts FIXME as compared to v21.8.3.44-lts + +#### Bug Fix +* Backported in [#27650](https://github.com/ClickHouse/ClickHouse/issues/27650): Fix crash during projection materialization when some parts contain missing columns. This fixes [#27512](https://github.com/ClickHouse/ClickHouse/issues/27512). [#27528](https://github.com/ClickHouse/ClickHouse/pull/27528) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#27625](https://github.com/ClickHouse/ClickHouse/issues/27625): Fixed underflow of the time value when constructing it from components. Closes [#27193](https://github.com/ClickHouse/ClickHouse/issues/27193). [#27605](https://github.com/ClickHouse/ClickHouse/pull/27605) ([Vasily Nemkov](https://github.com/Enmk)). +* Backported in [#27779](https://github.com/ClickHouse/ClickHouse/issues/27779): - Fix bug with aliased column in `Distributed` table. [#27652](https://github.com/ClickHouse/ClickHouse/pull/27652) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#27713](https://github.com/ClickHouse/ClickHouse/issues/27713): Fixed another case of `Unexpected merged part ... intersecting drop range ...` error. [#27656](https://github.com/ClickHouse/ClickHouse/pull/27656) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#27673](https://github.com/ClickHouse/ClickHouse/issues/27673): Fix postgresql table function resulting in non-closing connections. Closes [#26088](https://github.com/ClickHouse/ClickHouse/issues/26088). [#27662](https://github.com/ClickHouse/ClickHouse/pull/27662) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#27700](https://github.com/ClickHouse/ClickHouse/issues/27700): Fix bad type cast when functions like `arrayHas` are applied to arrays of LowCardinality of Nullable of different non-numeric types like `DateTime` and `DateTime64`. In previous versions bad cast occurs. In new version it will lead to exception. This closes [#26330](https://github.com/ClickHouse/ClickHouse/issues/26330). [#27682](https://github.com/ClickHouse/ClickHouse/pull/27682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#27747](https://github.com/ClickHouse/ClickHouse/issues/27747): Remove duplicated source files in CMakeLists.txt in arrow-cmake. [#27736](https://github.com/ClickHouse/ClickHouse/pull/27736) ([李扬](https://github.com/taiyang-li)). + diff --git a/docs/changelogs/v21.8.5.7-lts.md b/docs/changelogs/v21.8.5.7-lts.md new file mode 100644 index 00000000000..d78eb98b472 --- /dev/null +++ b/docs/changelogs/v21.8.5.7-lts.md @@ -0,0 +1,31 @@ +### ClickHouse release v21.8.5.7-lts FIXME as compared to v21.8.4.51-lts + +#### Improvement +* Backported in [#27893](https://github.com/ClickHouse/ClickHouse/issues/27893): Allow symlinks for library dictionaty path. [#27815](https://github.com/ClickHouse/ClickHouse/pull/27815) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#28154](https://github.com/ClickHouse/ClickHouse/issues/28154): Use Multipart copy upload for large S3 objects. [#27858](https://github.com/ClickHouse/ClickHouse/pull/27858) ([ianton-ru](https://github.com/ianton-ru)). +* Backported in [#28463](https://github.com/ClickHouse/ClickHouse/issues/28463): Do not allow creating StorageMaterializedPostgreSQL with bad arguments. Closes [#28423](https://github.com/ClickHouse/ClickHouse/issues/28423). [#28430](https://github.com/ClickHouse/ClickHouse/pull/28430) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Bug Fix +* Backported in [#28033](https://github.com/ClickHouse/ClickHouse/issues/28033): Bugfix for windowFunnel's "strict" mode. This fixes [#27469](https://github.com/ClickHouse/ClickHouse/issues/27469). [#27563](https://github.com/ClickHouse/ClickHouse/pull/27563) ([achimbab](https://github.com/achimbab)). +* Backported in [#27782](https://github.com/ClickHouse/ClickHouse/issues/27782): Fix column filtering with union distinct in subquery. Closes [#27578](https://github.com/ClickHouse/ClickHouse/issues/27578). [#27689](https://github.com/ClickHouse/ClickHouse/pull/27689) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#27867](https://github.com/ClickHouse/ClickHouse/issues/27867): Prevent crashes for some formats when NULL (tombstone) message was coming from Kafka. Closes [#19255](https://github.com/ClickHouse/ClickHouse/issues/19255). [#27794](https://github.com/ClickHouse/ClickHouse/pull/27794) ([filimonov](https://github.com/filimonov)). +* Backported in [#28346](https://github.com/ClickHouse/ClickHouse/issues/28346): Fix a rare bug in `DROP PART` which can lead to the error `Unexpected merged part intersects drop range`. [#27807](https://github.com/ClickHouse/ClickHouse/pull/27807) ([alesapin](https://github.com/alesapin)). +* Backported in [#27855](https://github.com/ClickHouse/ClickHouse/issues/27855): Fix a couple of bugs that may cause replicas to diverge. [#27808](https://github.com/ClickHouse/ClickHouse/pull/27808) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#27958](https://github.com/ClickHouse/ClickHouse/issues/27958): Fix selecting with extremes from a column of the type `LowCardinality(UUID)`. [#27918](https://github.com/ClickHouse/ClickHouse/pull/27918) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#27953](https://github.com/ClickHouse/ClickHouse/issues/27953): Check cluster name before creating Distributed table, do not allow to create a table with incorrect cluster name. Fixes [#27832](https://github.com/ClickHouse/ClickHouse/issues/27832). [#27927](https://github.com/ClickHouse/ClickHouse/pull/27927) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#28209](https://github.com/ClickHouse/ClickHouse/issues/28209): Fix cases, when read buffer fails with 'attempt to read after end of file'. Closes [#26149](https://github.com/ClickHouse/ClickHouse/issues/26149). [#28150](https://github.com/ClickHouse/ClickHouse/pull/28150) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + +#### Build/Testing/Packaging Improvement +* Backported in [#28031](https://github.com/ClickHouse/ClickHouse/issues/28031): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#27974](https://github.com/ClickHouse/ClickHouse/issues/27974): Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#28117](https://github.com/ClickHouse/ClickHouse/issues/28117): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#28180](https://github.com/ClickHouse/ClickHouse/issues/28180): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). +* Backported in [#28254](https://github.com/ClickHouse/ClickHouse/issues/28254): Fix incorrect behavior in `clickhouse-keeper` when list watches (`getChildren`) triggered with `set` requests for children. [#28190](https://github.com/ClickHouse/ClickHouse/pull/28190) ([alesapin](https://github.com/alesapin)). +* Backported in [#28342](https://github.com/ClickHouse/ClickHouse/issues/28342): Fix a rare bug in `clickhouse-keeper` when the client can receive a watch response before request-response. [#28197](https://github.com/ClickHouse/ClickHouse/pull/28197) ([alesapin](https://github.com/alesapin)). +* Backported in [#28263](https://github.com/ClickHouse/ClickHouse/issues/28263): Fix possible read of uninitialized memory for queries with `Nullable(LowCardinality)` type and extremes. Fixes [#28165](https://github.com/ClickHouse/ClickHouse/issues/28165). [#28205](https://github.com/ClickHouse/ClickHouse/pull/28205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#28292](https://github.com/ClickHouse/ClickHouse/issues/28292): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28402](https://github.com/ClickHouse/ClickHouse/issues/28402): Fix intersecting parts due to new part had been replaced with an empty part. [#28310](https://github.com/ClickHouse/ClickHouse/pull/28310) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.8.6.15-lts.md b/docs/changelogs/v21.8.6.15-lts.md new file mode 100644 index 00000000000..de38572d94a --- /dev/null +++ b/docs/changelogs/v21.8.6.15-lts.md @@ -0,0 +1,25 @@ +### ClickHouse release v21.8.6.15-lts FIXME as compared to v21.8.5.7-lts + +#### Improvement +* Backported in [#28899](https://github.com/ClickHouse/ClickHouse/issues/28899): Use real tmp file instead of predefined "rows_sources" for vertical merges. This avoids generating garbage directories in tmp disks. [#28299](https://github.com/ClickHouse/ClickHouse/pull/28299) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#28642](https://github.com/ClickHouse/ClickHouse/issues/28642): Fix strange sessions expiration logic in Keeper. Probably it should help in CI: https://clickhouse-test-reports.s3.yandex.net/0/6bd9b82141c98dcd7796fd9d08326831095ba519/stress_test_(debug).html#fail1. [#28519](https://github.com/ClickHouse/ClickHouse/pull/28519) ([alesapin](https://github.com/alesapin)). +* Backported in [#28718](https://github.com/ClickHouse/ClickHouse/issues/28718): To be added. Closes [#28529](https://github.com/ClickHouse/ClickHouse/issues/28529). [#28614](https://github.com/ClickHouse/ClickHouse/pull/28614) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Bug Fix +* Backported in [#27923](https://github.com/ClickHouse/ClickHouse/issues/27923): Fix PostgreSQL-style cast (`::` operator) with negative numbers. [#27876](https://github.com/ClickHouse/ClickHouse/pull/27876) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28753](https://github.com/ClickHouse/ClickHouse/issues/28753): Fix transformation of disjunctions chain to `IN` (controlled by settings `optimize_min_equality_disjunction_chain_length`) in distributed queries with settings `legacy_column_name_of_tuple_literal = 0`. [#28658](https://github.com/ClickHouse/ClickHouse/pull/28658) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#28644](https://github.com/ClickHouse/ClickHouse/issues/28644): Fix rare case when changes of `clickhouse-keeper` settings may lead to lost logs and server hung. [#28360](https://github.com/ClickHouse/ClickHouse/pull/28360) ([alesapin](https://github.com/alesapin)). +* Backported in [#28508](https://github.com/ClickHouse/ClickHouse/issues/28508): Fix lack of quotes for table names in MaterializedPostgreSQL engine. Closes [#28316](https://github.com/ClickHouse/ClickHouse/issues/28316). [#28433](https://github.com/ClickHouse/ClickHouse/pull/28433) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#28510](https://github.com/ClickHouse/ClickHouse/issues/28510): Fixed possible ZooKeeper watches leak on background processing of distributed DDL queue. Closes [#26036](https://github.com/ClickHouse/ClickHouse/issues/26036). [#28446](https://github.com/ClickHouse/ClickHouse/pull/28446) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#28573](https://github.com/ClickHouse/ClickHouse/issues/28573): Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)). +* Backported in [#28597](https://github.com/ClickHouse/ClickHouse/issues/28597): Fix `There is no subcolumn` error, while select from tables, which have `Nested` columns and scalar columns with dot in name and the same prefix as `Nested` (e.g. `n.id UInt32, n.arr1 Array(UInt64), n.arr2 Array(UInt64)`). [#28531](https://github.com/ClickHouse/ClickHouse/pull/28531) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28714](https://github.com/ClickHouse/ClickHouse/issues/28714): Add Settings.Names, Settings.Values aliases for system.processes table. [#28685](https://github.com/ClickHouse/ClickHouse/pull/28685) ([Vitaly Orlov](https://github.com/orloffv)). +* Backported in [#28741](https://github.com/ClickHouse/ClickHouse/issues/28741): Fix the coredump in the creation of distributed tables, when the parameters passed in are wrong. [#28686](https://github.com/ClickHouse/ClickHouse/pull/28686) ([Zhiyong Wang](https://github.com/ljcui)). +* Backported in [#28791](https://github.com/ClickHouse/ClickHouse/issues/28791): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)). +* Backported in [#28994](https://github.com/ClickHouse/ClickHouse/issues/28994): Fixed a race condition between `DROP PART` and `REPLACE/MOVE PARTITION` that might cause replicas to diverge in rare cases. [#28864](https://github.com/ClickHouse/ClickHouse/pull/28864) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#28948](https://github.com/ClickHouse/ClickHouse/issues/28948): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28930](https://github.com/ClickHouse/ClickHouse/issues/28930): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.8.7.22-lts.md b/docs/changelogs/v21.8.7.22-lts.md new file mode 100644 index 00000000000..92ba59f13cd --- /dev/null +++ b/docs/changelogs/v21.8.7.22-lts.md @@ -0,0 +1,8 @@ +### ClickHouse release v21.8.7.22-lts FIXME as compared to v21.8.6.15-lts + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#29121](https://github.com/ClickHouse/ClickHouse/issues/29121): Better check for connection usability and also catch any exception in RabbitMQ shutdown just in case. [#28797](https://github.com/ClickHouse/ClickHouse/pull/28797) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29027](https://github.com/ClickHouse/ClickHouse/issues/29027): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29193](https://github.com/ClickHouse/ClickHouse/issues/29193): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.8.8.29-lts.md b/docs/changelogs/v21.8.8.29-lts.md new file mode 100644 index 00000000000..199be63424c --- /dev/null +++ b/docs/changelogs/v21.8.8.29-lts.md @@ -0,0 +1,13 @@ +### ClickHouse release v21.8.8.29-lts FIXME as compared to v21.8.7.22-lts + +#### Bug Fix +* Backported in [#29128](https://github.com/ClickHouse/ClickHouse/issues/29128): Fix bug in `clickhouse-keeper-converter` which can lead to incorrect ZooKeeper log deserialization. [#29071](https://github.com/ClickHouse/ClickHouse/pull/29071) ([小路](https://github.com/nicelulu)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#29262](https://github.com/ClickHouse/ClickHouse/issues/29262): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#29106](https://github.com/ClickHouse/ClickHouse/issues/29106): Fix waiting for mutation with `mutations_sync=2`. [#28889](https://github.com/ClickHouse/ClickHouse/pull/28889) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29357](https://github.com/ClickHouse/ClickHouse/issues/29357): Fix possible `Table columns structure in ZooKeeper is different from local table structure` exception while recreating or creating new replicas of `ReplicatedMergeTree`, when one of table columns have default expressions with case-insensitive functions. [#29266](https://github.com/ClickHouse/ClickHouse/pull/29266) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#29447](https://github.com/ClickHouse/ClickHouse/issues/29447): Fix failed assertion in ReadBufferFromHDFS. Update libhdfs3 library to be able to run in tests in debug. Closes [#29251](https://github.com/ClickHouse/ClickHouse/issues/29251). Closes [#27814](https://github.com/ClickHouse/ClickHouse/issues/27814). [#29276](https://github.com/ClickHouse/ClickHouse/pull/29276) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29303](https://github.com/ClickHouse/ClickHouse/issues/29303): Fix connection timeouts (`send_timeout`/`receive_timeout`). [#29282](https://github.com/ClickHouse/ClickHouse/pull/29282) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.8.9.13-lts.md b/docs/changelogs/v21.8.9.13-lts.md new file mode 100644 index 00000000000..e4cd5f45b9b --- /dev/null +++ b/docs/changelogs/v21.8.9.13-lts.md @@ -0,0 +1,31 @@ +### ClickHouse release v21.8.9.13-lts FIXME as compared to v21.8.8.29-lts + +#### Improvement +* Backported in [#29941](https://github.com/ClickHouse/ClickHouse/issues/29941): Update zoneinfo files to 2021c. [#29925](https://github.com/ClickHouse/ClickHouse/pull/29925) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#29817](https://github.com/ClickHouse/ClickHouse/issues/29817): Allow using a materialized column as the sharding key in a distributed table even if `insert_allow_materialized_columns=0`:. [#28637](https://github.com/ClickHouse/ClickHouse/pull/28637) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#29973](https://github.com/ClickHouse/ClickHouse/issues/29973): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#29676](https://github.com/ClickHouse/ClickHouse/issues/29676): Fix vertical merges of projection parts. This fixes [#29253](https://github.com/ClickHouse/ClickHouse/issues/29253) . This PR also fixes several projection merge/mutation issues introduced in https://github.com/ClickHouse/ClickHouse/pull/25165. [#29337](https://github.com/ClickHouse/ClickHouse/pull/29337) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#29538](https://github.com/ClickHouse/ClickHouse/issues/29538): Fix possible `Block structure mismatch` for subqueries with pushed-down `HAVING` predicate. Fixes [#29010](https://github.com/ClickHouse/ClickHouse/issues/29010). [#29475](https://github.com/ClickHouse/ClickHouse/pull/29475) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29589](https://github.com/ClickHouse/ClickHouse/issues/29589): In ODBC bridge add retries for error Invalid cursor state. It is a retriable error. Closes [#29473](https://github.com/ClickHouse/ClickHouse/issues/29473). [#29518](https://github.com/ClickHouse/ClickHouse/pull/29518) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29593](https://github.com/ClickHouse/ClickHouse/issues/29593): Fix bug in check `pathStartsWith` becuase there was bug with the usage of `std::mismatch`: ` The behavior is undefined if the second range is shorter than the first range.`. [#29531](https://github.com/ClickHouse/ClickHouse/pull/29531) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29628](https://github.com/ClickHouse/ClickHouse/issues/29628): Fix rare segfault in `ALTER MODIFY` query when using incorrect table identifier in `DEFAULT` expression like `x.y.z...` Fixes [#29184](https://github.com/ClickHouse/ClickHouse/issues/29184). [#29573](https://github.com/ClickHouse/ClickHouse/pull/29573) ([alesapin](https://github.com/alesapin)). +* Backported in [#29752](https://github.com/ClickHouse/ClickHouse/issues/29752): Condition in filter predicate could be lost after push-down optimisation. [#29625](https://github.com/ClickHouse/ClickHouse/pull/29625) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29840](https://github.com/ClickHouse/ClickHouse/issues/29840): Fixed incorrect behaviour of setting `materialized_postgresql_tables_list` at server restart. Found in [#28529](https://github.com/ClickHouse/ClickHouse/issues/28529). [#29686](https://github.com/ClickHouse/ClickHouse/pull/29686) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29847](https://github.com/ClickHouse/ClickHouse/issues/29847): Fix concurrent access to `LowCardinality` during `GROUP BY` (leads to SIGSEGV). [#29782](https://github.com/ClickHouse/ClickHouse/pull/29782) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29911](https://github.com/ClickHouse/ClickHouse/issues/29911): Fix bad cast in `ATTACH TABLE ... FROM 'path'` query when non-string literal is used instead of path. It may lead to reading of uninitialized memory. [#29790](https://github.com/ClickHouse/ClickHouse/pull/29790) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#29867](https://github.com/ClickHouse/ClickHouse/issues/29867): Avoid `Timeout exceeded: elapsed 18446744073.709553 seconds` error that might happen in extremely rare cases, presumably due to some bug in kernel. Fixes [#29154](https://github.com/ClickHouse/ClickHouse/issues/29154). [#29811](https://github.com/ClickHouse/ClickHouse/pull/29811) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#29877](https://github.com/ClickHouse/ClickHouse/issues/29877): Fix system tables recreation check (fails to detect changes in enum values). [#29857](https://github.com/ClickHouse/ClickHouse/pull/29857) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30210](https://github.com/ClickHouse/ClickHouse/issues/30210): Fix possible data-race between `FileChecker` and `StorageLog`/`StorageStripeLog`. [#29959](https://github.com/ClickHouse/ClickHouse/pull/29959) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30069](https://github.com/ClickHouse/ClickHouse/issues/30069): Fix crash of sample by `tuple()`, closes [#30004](https://github.com/ClickHouse/ClickHouse/issues/30004). [#30016](https://github.com/ClickHouse/ClickHouse/pull/30016) ([flynn](https://github.com/ucasfl)). +* Backported in [#30125](https://github.com/ClickHouse/ClickHouse/issues/30125): Dropped `Memory` database might reappear after server restart, it's fixed ([#29795](https://github.com/ClickHouse/ClickHouse/issues/29795)). Also added `force_remove_data_recursively_on_drop` setting as a workaround for `Directory not empty` error when dropping `Ordinary` database (because it's not possible to remove data leftovers manually in cloud environment). [#30054](https://github.com/ClickHouse/ClickHouse/pull/30054) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30263](https://github.com/ClickHouse/ClickHouse/issues/30263): FlatDictionary, HashedDictionary fix bytes_allocated calculation for nullable attributes. [#30238](https://github.com/ClickHouse/ClickHouse/pull/30238) ([Maksim Kita](https://github.com/kitaisreal)). + +#### NO CL CATEGORY + +* Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30187](https://github.com/ClickHouse/ClickHouse/pull/30187) ([Raúl Marín](https://github.com/Algunenano)). + diff --git a/docs/changelogs/v21.9.1.8000-prestable.md b/docs/changelogs/v21.9.1.8000-prestable.md new file mode 100644 index 00000000000..cee357658d2 --- /dev/null +++ b/docs/changelogs/v21.9.1.8000-prestable.md @@ -0,0 +1,190 @@ +### ClickHouse release v21.9.1.8000-prestable FIXME as compared to v21.8.1.7409-prestable + +#### Backward Incompatible Change +* Fix the issue that in case of some sophisticated query with column aliases identical to the names of expressions, bad cast may happen. This fixes [#25447](https://github.com/ClickHouse/ClickHouse/issues/25447). This fixes [#26914](https://github.com/ClickHouse/ClickHouse/issues/26914). This fix may introduce backward incompatibility: if there are different expressions with identical names, exception will be thrown. It may break some rare cases when `enable_optimize_predicate_expression` is set. [#26639](https://github.com/ClickHouse/ClickHouse/pull/26639) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Under clickhouse-local, always treat local addresses with a port as remote. [#26736](https://github.com/ClickHouse/ClickHouse/pull/26736) ([Raúl Marín](https://github.com/Algunenano)). +* Do not allow to apply parametric aggregate function with `-Merge` combinator to aggregate function state if state was produced by aggregate function with different parameters. For example, state of `fooState(42)(x)` cannot be finalized with `fooMerge(s)` or `fooMerge(123)(s)`, parameters must be specified explicitly like `fooMerge(42)(s)` and must be equal. It does not affect some special aggregate functions like `quantile` and `sequence*` that use parameters for finalization only. [#26847](https://github.com/ClickHouse/ClickHouse/pull/26847) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Do not output trailing zeros in text representation of `Decimal` types. Example: `1.23` will be printed instead of `1.230000` for decimal with scale 6. This closes [#15794](https://github.com/ClickHouse/ClickHouse/issues/15794). It may introduce slight incompatibility if your applications somehow relied on the trailing zeros. Serialization in output formats can be controlled with the setting `output_format_decimal_trailing_zeros`. Implementation of `toString` and casting to String is changed unconditionally. [#27680](https://github.com/ClickHouse/ClickHouse/pull/27680) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* Implement window function `nth_value(expr, N)` that returns the value of the Nth row of the window frame. [#26334](https://github.com/ClickHouse/ClickHouse/pull/26334) ([Zuo, RuoYu](https://github.com/ryzuo)). +* - Add `REPLACE GRANT` feature. [#26384](https://github.com/ClickHouse/ClickHouse/pull/26384) ([Caspian](https://github.com/Cas-pian)). +* Functions that return (initial_)query_id of the current query. This closes [#23682](https://github.com/ClickHouse/ClickHouse/issues/23682). [#26410](https://github.com/ClickHouse/ClickHouse/pull/26410) ([Alexey Boykov](https://github.com/mathalex)). +* Introduce syntax for here documents. Example `SELECT $doc$VALUE$doc$`. [#26671](https://github.com/ClickHouse/ClickHouse/pull/26671) ([Maksim Kita](https://github.com/kitaisreal)). +* New functions `currentProfiles()`, `enabledProfiles()`, `defaultProfiles()`. [#26714](https://github.com/ClickHouse/ClickHouse/pull/26714) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add new functions `currentRoles()`, `enabledRoles()`, `defaultRoles()`. [#26780](https://github.com/ClickHouse/ClickHouse/pull/26780) ([Vitaly Baranov](https://github.com/vitlibar)). +* Supported cluster macros inside table functions 'cluster' and 'clusterAllReplicas'. [#26913](https://github.com/ClickHouse/ClickHouse/pull/26913) ([Vadim Volodin](https://github.com/PolyProgrammist)). +* Added support for custom query for MySQL, PostgreSQL, ClickHouse, JDBC, Cassandra dictionary source. Closes [#1270](https://github.com/ClickHouse/ClickHouse/issues/1270). [#26995](https://github.com/ClickHouse/ClickHouse/pull/26995) ([Maksim Kita](https://github.com/kitaisreal)). +* add column default_database to system.users. [#27054](https://github.com/ClickHouse/ClickHouse/pull/27054) ([kevin wan](https://github.com/MaxWk)). +* Added `bitmapSubsetOffsetLimit(bitmap, offset, cardinality_limit)` function. It creates a subset of bitmap limit the results to `cardinality_limit` with offset of `offset`. [#27234](https://github.com/ClickHouse/ClickHouse/pull/27234) ([DHBin](https://github.com/DHBin)). +* Add support for `bzip2` compression method for import/export. Closes [#22428](https://github.com/ClickHouse/ClickHouse/issues/22428). [#27377](https://github.com/ClickHouse/ClickHouse/pull/27377) ([Nikolay Degterinsky](https://github.com/evillique)). +* - Add replicated storage of user, roles, row policies, quotas and settings profiles through ZooKeeper (experimental). [#27426](https://github.com/ClickHouse/ClickHouse/pull/27426) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Add "tupleToNameValuePairs", a function that turns a named tuple into an array of pairs. [#27505](https://github.com/ClickHouse/ClickHouse/pull/27505) ([Braulio Valdivielso Martínez](https://github.com/BraulioVM)). +* Enable using constants from with and select in aggregate function parameters. Close [#10945](https://github.com/ClickHouse/ClickHouse/issues/10945). [#27531](https://github.com/ClickHouse/ClickHouse/pull/27531) ([abel-cheng](https://github.com/abel-cheng)). +* Added ComplexKeyRangeHashed dictionary. Closes [#22029](https://github.com/ClickHouse/ClickHouse/issues/22029). [#27629](https://github.com/ClickHouse/ClickHouse/pull/27629) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Performance Improvement +* Compile aggregate functions `groupBitOr`, `groupBitAnd`, `groupBitXor`. [#26161](https://github.com/ClickHouse/ClickHouse/pull/26161) ([Maksim Kita](https://github.com/kitaisreal)). +* Compile columns with `Enum` types. [#26237](https://github.com/ClickHouse/ClickHouse/pull/26237) ([Maksim Kita](https://github.com/kitaisreal)). +* - Vectorize the SUM of Nullable integer types with native representation ([David Manzanares](https://github.com/davidmanzanares), [Raúl Marín](https://github.com/Algunenano)). [#26248](https://github.com/ClickHouse/ClickHouse/pull/26248) ([Raúl Marín](https://github.com/Algunenano)). +* Don't build sets for indices when analyzing a query. [#26365](https://github.com/ClickHouse/ClickHouse/pull/26365) ([Raúl Marín](https://github.com/Algunenano)). +* Improve latency of short queries, that require reading from tables with large number of columns. [#26371](https://github.com/ClickHouse/ClickHouse/pull/26371) ([Anton Popov](https://github.com/CurtizJ)). +* Share file descriptors in concurrent reads of the same files. There is no noticeable performance difference on Linux. But the number of opened files will be significantly (10..100 times) lower on typical servers and it makes operations easier. See [#26214](https://github.com/ClickHouse/ClickHouse/issues/26214). [#26768](https://github.com/ClickHouse/ClickHouse/pull/26768) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Specialize date time related comparison to achieve better performance. This fixes [#27083](https://github.com/ClickHouse/ClickHouse/issues/27083) . [#27122](https://github.com/ClickHouse/ClickHouse/pull/27122) ([Amos Bird](https://github.com/amosbird)). +* Improve the performance of fast queries when `max_execution_time=0` by reducing the number of `clock_gettime` system calls. [#27325](https://github.com/ClickHouse/ClickHouse/pull/27325) ([filimonov](https://github.com/filimonov)). +* Less number of `clock_gettime` syscalls that may lead to performance improvement for some types of fast queries. [#27492](https://github.com/ClickHouse/ClickHouse/pull/27492) ([filimonov](https://github.com/filimonov)). + +#### Improvement +* Add error id (like `BAD_ARGUMENTS`) to exception messages. This closes [#25862](https://github.com/ClickHouse/ClickHouse/issues/25862). [#26172](https://github.com/ClickHouse/ClickHouse/pull/26172) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove GLOBAL keyword for IN when scalar function is passed. In previous versions, if user specified `GLOBAL IN f(x)` exception was thrown. [#26217](https://github.com/ClickHouse/ClickHouse/pull/26217) ([Amos Bird](https://github.com/amosbird)). +* Apply aggressive IN index analysis for projections so that better projection candidate can be selected. [#26218](https://github.com/ClickHouse/ClickHouse/pull/26218) ([Amos Bird](https://github.com/amosbird)). +* convert timestamp and timestamptz data types to DateTime64 in postgres engine. [#26234](https://github.com/ClickHouse/ClickHouse/pull/26234) ([jasine](https://github.com/jasine)). +* Check for non-deterministic functions in keys, including constant expressions like `now()`, `today()`. This closes [#25875](https://github.com/ClickHouse/ClickHouse/issues/25875). This closes [#11333](https://github.com/ClickHouse/ClickHouse/issues/11333). [#26235](https://github.com/ClickHouse/ClickHouse/pull/26235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't throw exception when querying `system.detached_parts` table if there is custom disk configuration and `detached` directory does not exist on some disks. This closes [#26078](https://github.com/ClickHouse/ClickHouse/issues/26078). [#26236](https://github.com/ClickHouse/ClickHouse/pull/26236) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add information about column sizes in `system.columns` table for `Log` and `TinyLog` tables. This closes [#9001](https://github.com/ClickHouse/ClickHouse/issues/9001). [#26241](https://github.com/ClickHouse/ClickHouse/pull/26241) ([Nikolay Degterinsky](https://github.com/evillique)). +* Added `output_format_avro_string_column_pattern` setting to put specified String columns to Avro as string instead of default bytes. Implements [#22414](https://github.com/ClickHouse/ClickHouse/issues/22414). [#26245](https://github.com/ClickHouse/ClickHouse/pull/26245) ([Ilya Golshtein](https://github.com/ilejn)). +* - Add `system.warnings` table to collect warnings about server configuration. [#26246](https://github.com/ClickHouse/ClickHouse/pull/26246) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Check hash function at table creation, not at sampling. Add settings in MergeTreeSettings, if someone create a table with incorrect sampling column but sampling never be used, disable this settings for starting the server without exception. [#26256](https://github.com/ClickHouse/ClickHouse/pull/26256) ([zhaoyu](https://github.com/zxc111)). +* Make `toTimeZone` monotonicity when timeZone is a constant value to support partition puring when use sql like:. [#26261](https://github.com/ClickHouse/ClickHouse/pull/26261) ([huangzhaowei](https://github.com/SaintBacchus)). +* - When client connect to server, he receives information about all warnings that are already were collected by server. (It can be disabled by using option `--no-warnings`). [#26282](https://github.com/ClickHouse/ClickHouse/pull/26282) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Add a setting `function_range_max_elements_in_block` to tune the safety threshold for data volume generated by function `range`. This closes [#26303](https://github.com/ClickHouse/ClickHouse/issues/26303). [#26305](https://github.com/ClickHouse/ClickHouse/pull/26305) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Control the execution period of clear old temporary directories by parameter with default value. [#26212](https://github.com/ClickHouse/ClickHouse/issues/26212). [#26313](https://github.com/ClickHouse/ClickHouse/pull/26313) ([fastio](https://github.com/fastio)). +* Allow to reuse connections of shards among different clusters. It also avoids creating new connections when using `cluster` table function. [#26318](https://github.com/ClickHouse/ClickHouse/pull/26318) ([Amos Bird](https://github.com/amosbird)). +* Add events to profile calls to sleep / sleepEachRow. [#26320](https://github.com/ClickHouse/ClickHouse/pull/26320) ([Raúl Marín](https://github.com/Algunenano)). +* Save server address in history URLs in web UI if it differs from the origin of web UI. This closes [#26044](https://github.com/ClickHouse/ClickHouse/issues/26044). [#26322](https://github.com/ClickHouse/ClickHouse/pull/26322) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add ability to set Distributed directory monitor settings via CREATE TABLE (i.e. `CREATE TABLE dist (key Int) Engine=Distributed(cluster, db, table) SETTINGS monitor_batch_inserts=1` and similar). [#26336](https://github.com/ClickHouse/ClickHouse/pull/26336) ([Azat Khuzhin](https://github.com/azat)). +* Fix behaviour with non-existing host in user allowed host list. [#26368](https://github.com/ClickHouse/ClickHouse/pull/26368) ([ianton-ru](https://github.com/ianton-ru)). +* Added comments for the code written in https://github.com/ClickHouse/ClickHouse/pull/24206; the code has been improved in several places. [#26377](https://github.com/ClickHouse/ClickHouse/pull/26377) ([Vitaly Baranov](https://github.com/vitlibar)). +* Enable `use_hedged_requests` setting that allows to mitigate tail latencies on large clusters. [#26380](https://github.com/ClickHouse/ClickHouse/pull/26380) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Updated protobuf to 3.17.3. Changelogs are available on https://github.com/protocolbuffers/protobuf/releases. [#26424](https://github.com/ClickHouse/ClickHouse/pull/26424) ([Ilya Yatsishin](https://github.com/qoega)). +* After https://github.com/ClickHouse/ClickHouse/pull/26377. Encryption algorithm now should be specified explicitly if it's not default (`aes_128_ctr`):. [#26465](https://github.com/ClickHouse/ClickHouse/pull/26465) ([Vitaly Baranov](https://github.com/vitlibar)). +* Apply `LIMIT` on the shards for queries like `SELECT * FROM dist ORDER BY key LIMIT 10` w/ `distributed_push_down_limit=1`. Avoid running `Distinct`/`LIMIT BY` steps for queries like `SELECT DISTINCT shading_key FROM dist ORDER BY key`. Now `distributed_push_down_limit` is respected by `optimize_distributed_group_by_sharding_key` optimization. [#26466](https://github.com/ClickHouse/ClickHouse/pull/26466) ([Azat Khuzhin](https://github.com/azat)). +* - Set client query kind for mysql and postgresql handler. [#26498](https://github.com/ClickHouse/ClickHouse/pull/26498) ([anneji-dev](https://github.com/anneji-dev)). +* Executable dictionaries (ExecutableDictionarySource, ExecutablePoolDictionarySource) enable creation with DDL query using clickhouse-local. Closes [#22355](https://github.com/ClickHouse/ClickHouse/issues/22355). [#26510](https://github.com/ClickHouse/ClickHouse/pull/26510) ([Maksim Kita](https://github.com/kitaisreal)). +* Add round-robin support for clickhouse-benchmark (it does not differ from the regular multi host/port run except for statistics report). [#26607](https://github.com/ClickHouse/ClickHouse/pull/26607) ([Azat Khuzhin](https://github.com/azat)). +* Improve the high performance machine to use the kafka engine. and it can recuce the query node work load. [#26642](https://github.com/ClickHouse/ClickHouse/pull/26642) ([feihengye](https://github.com/feihengye)). +* Avoid hanging clickhouse-benchmark if connection fails (i.e. on EMFILE). [#26656](https://github.com/ClickHouse/ClickHouse/pull/26656) ([Azat Khuzhin](https://github.com/azat)). +* Fix excessive (x2) connect attempts with skip_unavailable_shards. [#26658](https://github.com/ClickHouse/ClickHouse/pull/26658) ([Azat Khuzhin](https://github.com/azat)). +* - `mapPopulatesSeries` function supports `Map` type. [#26663](https://github.com/ClickHouse/ClickHouse/pull/26663) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Improve handling of KILL QUERY requests. [#26675](https://github.com/ClickHouse/ClickHouse/pull/26675) ([Raúl Marín](https://github.com/Algunenano)). +* SET PROFILE now applies constraints too if they're set for a passed profile. [#26730](https://github.com/ClickHouse/ClickHouse/pull/26730) ([Vitaly Baranov](https://github.com/vitlibar)). +* Support multiple keys for encrypted disk. Display error message if the key is probably wrong. (see https://github.com/ClickHouse/ClickHouse/pull/26465#issuecomment-882015970). [#26733](https://github.com/ClickHouse/ClickHouse/pull/26733) ([Vitaly Baranov](https://github.com/vitlibar)). +* remove uncessary exception thrown. [#26740](https://github.com/ClickHouse/ClickHouse/pull/26740) ([Caspian](https://github.com/Cas-pian)). +* Watchdog is disabled in docker by default. Fix for not handling ctrl+c. [#26757](https://github.com/ClickHouse/ClickHouse/pull/26757) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Changing default roles affects new sessions only. [#26759](https://github.com/ClickHouse/ClickHouse/pull/26759) ([Vitaly Baranov](https://github.com/vitlibar)). +* Less verbose internal RocksDB logs. This closes [#26252](https://github.com/ClickHouse/ClickHouse/issues/26252). [#26789](https://github.com/ClickHouse/ClickHouse/pull/26789) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Expose rocksdb statistics via system.rocksdb table. Read rocksdb options from ClickHouse config (`rocksdb`/`rocksdb_TABLE` keys). [#26821](https://github.com/ClickHouse/ClickHouse/pull/26821) ([Azat Khuzhin](https://github.com/azat)). +* Updated extractAllGroupsHorizontal - upper limit on the number of matches per row can be set via optional third argument. ... [#26961](https://github.com/ClickHouse/ClickHouse/pull/26961) ([Vasily Nemkov](https://github.com/Enmk)). +* Now functions can be shard-level constants, which means if it's executed in the context of some distributed table, it generates a normal column, otherwise it produces a constant value. Notable functions are: `hostName()`, `tcpPort()`, `version()`, `buildId()`, `uptime()`, etc. [#27020](https://github.com/ClickHouse/ClickHouse/pull/27020) ([Amos Bird](https://github.com/amosbird)). +* * Merge join correctly handles empty set in the right. [#27078](https://github.com/ClickHouse/ClickHouse/pull/27078) ([Vladimir C](https://github.com/vdimir)). +* Improve compatibility with non-whole-minute timezone offsets. [#27080](https://github.com/ClickHouse/ClickHouse/pull/27080) ([Raúl Marín](https://github.com/Algunenano)). +* Enable distributed_push_down_limit by default. [#27104](https://github.com/ClickHouse/ClickHouse/pull/27104) ([Azat Khuzhin](https://github.com/azat)). +* Improved the existence condition judgment and empty string node judgment when clickhouse-keeper creates znode. [#27125](https://github.com/ClickHouse/ClickHouse/pull/27125) ([小路](https://github.com/nicelulu)). +* Add compression for `INTO OUTFILE` that automatically choose compression algorithm. Closes [#3473](https://github.com/ClickHouse/ClickHouse/issues/3473). [#27134](https://github.com/ClickHouse/ClickHouse/pull/27134) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* add a new metric called MaxPushedDDLEntryID which is the maximum ddl entry id that current node push to zookeeper. [#27174](https://github.com/ClickHouse/ClickHouse/pull/27174) ([Fuwang Hu](https://github.com/fuwhu)). +* Allow to pass query settings via server URI in Web UI. [#27177](https://github.com/ClickHouse/ClickHouse/pull/27177) ([kolsys](https://github.com/kolsys)). +* Added columns `replica_is_active` that maps replica name to is replica active status to table `system.replicas`. Closes [#27138](https://github.com/ClickHouse/ClickHouse/issues/27138). [#27180](https://github.com/ClickHouse/ClickHouse/pull/27180) ([Maksim Kita](https://github.com/kitaisreal)). +* Try recording `query_kind` even when query fails to start. [#27182](https://github.com/ClickHouse/ClickHouse/pull/27182) ([Amos Bird](https://github.com/amosbird)). +* Mark window functions as ready for general use. Remove the `allow_experimental_window_functions` setting. [#27184](https://github.com/ClickHouse/ClickHouse/pull/27184) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Memory client in client. [#27191](https://github.com/ClickHouse/ClickHouse/pull/27191) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Support schema for postgres database engine. Closes [#27166](https://github.com/ClickHouse/ClickHouse/issues/27166). [#27198](https://github.com/ClickHouse/ClickHouse/pull/27198) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Split global mutex into individual regexp construction. This helps avoid huge regexp construction blocking other related threads. Not sure how to proper test the improvement. [#27211](https://github.com/ClickHouse/ClickHouse/pull/27211) ([Amos Bird](https://github.com/amosbird)). +* Add 10 seconds cache for S3 proxy resolver. [#27216](https://github.com/ClickHouse/ClickHouse/pull/27216) ([ianton-ru](https://github.com/ianton-ru)). +* Add new index data skipping minmax index format for proper Nullable support. [#27250](https://github.com/ClickHouse/ClickHouse/pull/27250) ([Azat Khuzhin](https://github.com/azat)). +* Memory consumed by bitmap aggregate functions now is taken into account for memory limits. This closes [#26555](https://github.com/ClickHouse/ClickHouse/issues/26555). [#27252](https://github.com/ClickHouse/ClickHouse/pull/27252) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add two settings `max_hyperscan_regexp_length` and `max_hyperscan_regexp_total_length` to prevent huge regexp being used in hyperscan related functions, such as `multiMatchAny`. [#27378](https://github.com/ClickHouse/ClickHouse/pull/27378) ([Amos Bird](https://github.com/amosbird)). +* Add setting `log_formatted_queries` to log additional formatted query into `system.query_log`. It's useful for normalized query analysis because functions like `normalizeQuery` and `normalizeQueryKeepNames` don't parse/format queries in order to achieve better performance. [#27380](https://github.com/ClickHouse/ClickHouse/pull/27380) ([Amos Bird](https://github.com/amosbird)). +* Add Cast function for internal usage, which will not preserve type nullability, but non-internal cast will preserve according to setting cast_keep_nullable. Closes [#12636](https://github.com/ClickHouse/ClickHouse/issues/12636). [#27382](https://github.com/ClickHouse/ClickHouse/pull/27382) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Send response with error message if HTTP port is not set and user tries to send HTTP request to TCP port. [#27385](https://github.com/ClickHouse/ClickHouse/pull/27385) ([Braulio Valdivielso Martínez](https://github.com/BraulioVM)). +* Use bytes instead of strings for binary data in the GRPC protocol. [#27431](https://github.com/ClickHouse/ClickHouse/pull/27431) ([Vitaly Baranov](https://github.com/vitlibar)). +* Log client IP address if authentication fails. [#27514](https://github.com/ClickHouse/ClickHouse/pull/27514) ([Misko Lee](https://github.com/imiskolee)). +* Disable arrayJoin on partition expressions. [#27648](https://github.com/ClickHouse/ClickHouse/pull/27648) ([Raúl Marín](https://github.com/Algunenano)). +* - Add `FROM INFILE` command. [#27655](https://github.com/ClickHouse/ClickHouse/pull/27655) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Enables query parameters to be passed in the body of http requests. [#27706](https://github.com/ClickHouse/ClickHouse/pull/27706) ([Hermano Lustosa](https://github.com/hllustosa)). +* Remove duplicate index analysis and avoid possible invalid limit checks during projection analysis. [#27742](https://github.com/ClickHouse/ClickHouse/pull/27742) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix +* Fix potential crash if more than one `untuple` expression is used. [#26179](https://github.com/ClickHouse/ClickHouse/pull/26179) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix logical error on join with totals, close [#26017](https://github.com/ClickHouse/ClickHouse/issues/26017). [#26250](https://github.com/ClickHouse/ClickHouse/pull/26250) ([Vladimir C](https://github.com/vdimir)). +* Fix zstd decompression in case there are escape sequences at the end of internal buffer. Closes [#26013](https://github.com/ClickHouse/ClickHouse/issues/26013). [#26314](https://github.com/ClickHouse/ClickHouse/pull/26314) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed rare bug in lost replica recovery that may cause replicas to diverge. [#26321](https://github.com/ClickHouse/ClickHouse/pull/26321) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix `optimize_distributed_group_by_sharding_key` for multiple columns (leads to incorrect result w/ `optimize_skip_unused_shards=1`/`allow_nondeterministic_optimize_skip_unused_shards=1` and multiple columns in sharding key expression). [#26353](https://github.com/ClickHouse/ClickHouse/pull/26353) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible crash when login as dropped user. This PR fixes [#26073](https://github.com/ClickHouse/ClickHouse/issues/26073). [#26363](https://github.com/ClickHouse/ClickHouse/pull/26363) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix infinite non joined block stream in `partial_merge_join` close [#26325](https://github.com/ClickHouse/ClickHouse/issues/26325). [#26374](https://github.com/ClickHouse/ClickHouse/pull/26374) ([Vladimir C](https://github.com/vdimir)). +* Now, scalar subquery always returns `Nullable` result if it's type can be `Nullable`. It is needed because in case of empty subquery it's result should be `Null`. Previously, it was possible to get error about incompatible types (type deduction does not execute scalar subquery, and it could use not-nullable type). Scalar subquery with empty result which can't be converted to `Nullable` (like `Array` or `Tuple`) now throws error. Fixes [#25411](https://github.com/ClickHouse/ClickHouse/issues/25411). [#26423](https://github.com/ClickHouse/ClickHouse/pull/26423) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix some fuzzed msan crash. Fixes [#22517](https://github.com/ClickHouse/ClickHouse/issues/22517). [#26428](https://github.com/ClickHouse/ClickHouse/pull/26428) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix broken name resolution after rewriting column aliases. This fixes [#26432](https://github.com/ClickHouse/ClickHouse/issues/26432). [#26475](https://github.com/ClickHouse/ClickHouse/pull/26475) ([Amos Bird](https://github.com/amosbird)). +* Fix issues with `CREATE DICTIONARY` query if dictionary name or database name was quoted. Closes [#26491](https://github.com/ClickHouse/ClickHouse/issues/26491). [#26508](https://github.com/ClickHouse/ClickHouse/pull/26508) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix crash in rabbitmq shutdown in case rabbitmq setup was not started. Closes [#26504](https://github.com/ClickHouse/ClickHouse/issues/26504). [#26529](https://github.com/ClickHouse/ClickHouse/pull/26529) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update `chown` cmd check in clickhouse-server docker entrypoint. It fixes the bug that cluster pod restart failed (or timeout) on kubernetes. [#26545](https://github.com/ClickHouse/ClickHouse/pull/26545) ([Ky Li](https://github.com/Kylinrix)). +* Fix incorrect function names of groupBitmapAnd/Or/Xor. This fixes. [#26557](https://github.com/ClickHouse/ClickHouse/pull/26557) ([Amos Bird](https://github.com/amosbird)). +* Fix history file conversion if file is empty. [#26589](https://github.com/ClickHouse/ClickHouse/pull/26589) ([Azat Khuzhin](https://github.com/azat)). +* Fix potential nullptr dereference in window functions. This fixes [#25276](https://github.com/ClickHouse/ClickHouse/issues/25276). [#26668](https://github.com/ClickHouse/ClickHouse/pull/26668) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ParallelFormattingOutputFormat: Use mutex to handle the join to the collector_thread (https://github.com/ClickHouse/ClickHouse/issues/26694). [#26703](https://github.com/ClickHouse/ClickHouse/pull/26703) ([Raúl Marín](https://github.com/Algunenano)). +* Sometimes SET ROLE could work incorrectly, this PR fixes that. [#26707](https://github.com/ClickHouse/ClickHouse/pull/26707) ([Vitaly Baranov](https://github.com/vitlibar)). +* Do not remove data on ReplicatedMergeTree table shutdown to avoid creating data to metadata inconsistency. [#26716](https://github.com/ClickHouse/ClickHouse/pull/26716) ([nvartolomei](https://github.com/nvartolomei)). +* Add `event_time_microseconds` value for `REMOVE_PART` in `system.part_log`. In previous versions is was not set. [#26720](https://github.com/ClickHouse/ClickHouse/pull/26720) ([Azat Khuzhin](https://github.com/azat)). +* Aggregate function parameters might be lost when applying some combinators causing exceptions like `Conversion from AggregateFunction(topKArray, Array(String)) to AggregateFunction(topKArray(10), Array(String)) is not supported`. It's fixed. Fixes [#26196](https://github.com/ClickHouse/ClickHouse/issues/26196) and [#26433](https://github.com/ClickHouse/ClickHouse/issues/26433). [#26814](https://github.com/ClickHouse/ClickHouse/pull/26814) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix library-bridge ids load. [#26834](https://github.com/ClickHouse/ClickHouse/pull/26834) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix error `Missing columns: 'xxx'` when `DEFAULT` column references other non materialized column without `DEFAULT` expression. Fixes [#26591](https://github.com/ClickHouse/ClickHouse/issues/26591). [#26900](https://github.com/ClickHouse/ClickHouse/pull/26900) ([alesapin](https://github.com/alesapin)). +* Fix reading of custom TLDs (stops processing with lower buffer or bigger file). [#26948](https://github.com/ClickHouse/ClickHouse/pull/26948) ([Azat Khuzhin](https://github.com/azat)). +* Fix "Unknown column name" error with multiple JOINs in some cases, close [#26899](https://github.com/ClickHouse/ClickHouse/issues/26899). [#26957](https://github.com/ClickHouse/ClickHouse/pull/26957) ([Vladimir C](https://github.com/vdimir)). +* Now partition ID in queries like `ALTER TABLE ... PARTITION ID xxx` validates for correctness. Fixes [#25718](https://github.com/ClickHouse/ClickHouse/issues/25718). [#26963](https://github.com/ClickHouse/ClickHouse/pull/26963) ([alesapin](https://github.com/alesapin)). +* [RFC] Fix possible mutation stack due to race with DROP_RANGE. [#27002](https://github.com/ClickHouse/ClickHouse/pull/27002) ([Azat Khuzhin](https://github.com/azat)). +* Fixed `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache` configuration parsing. Options `allow_read_expired_keys`, `max_update_queue_size`, `update_queue_push_timeout_milliseconds`, `query_wait_timeout_milliseconds` were not parsed for dictionaries with non `cache` type. [#27032](https://github.com/ClickHouse/ClickHouse/pull/27032) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix synchronization in GRPCServer This PR fixes [#27024](https://github.com/ClickHouse/ClickHouse/issues/27024). [#27064](https://github.com/ClickHouse/ClickHouse/pull/27064) ([Vitaly Baranov](https://github.com/vitlibar)). +* - Fix uninitialized memory in functions `multiSearch*` with empty array, close [#27169](https://github.com/ClickHouse/ClickHouse/issues/27169). [#27181](https://github.com/ClickHouse/ClickHouse/pull/27181) ([Vladimir C](https://github.com/vdimir)). +* In rare cases `system.detached_parts` table might contain incorrect information for some parts, it's fixed. Fixes [#27114](https://github.com/ClickHouse/ClickHouse/issues/27114). [#27183](https://github.com/ClickHouse/ClickHouse/pull/27183) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix on-disk format breakage for secondary indices over Nullable column (no stable release had been affected). [#27197](https://github.com/ClickHouse/ClickHouse/pull/27197) ([Azat Khuzhin](https://github.com/azat)). +* Fix column structure in merge join, close [#27091](https://github.com/ClickHouse/ClickHouse/issues/27091). [#27217](https://github.com/ClickHouse/ClickHouse/pull/27217) ([Vladimir C](https://github.com/vdimir)). +* In case of ambiguity, lambda functions prefer its arguments to other aliases or identifiers. [#27235](https://github.com/ClickHouse/ClickHouse/pull/27235) ([Raúl Marín](https://github.com/Algunenano)). +* Fix mutation stuck on invalid partitions in non-replicated MergeTree. [#27248](https://github.com/ClickHouse/ClickHouse/pull/27248) ([Azat Khuzhin](https://github.com/azat)). +* Fix `distributed_group_by_no_merge=2`+`distributed_push_down_limit=1` or `optimize_distributed_group_by_sharding_key=1` with `LIMIT BY` and `LIMIT OFFSET`. [#27249](https://github.com/ClickHouse/ClickHouse/pull/27249) ([Azat Khuzhin](https://github.com/azat)). +* Fix errors like `Expected ColumnLowCardinality, gotUInt8` or `Bad cast from type DB::ColumnVector to DB::ColumnLowCardinality` for some queries with `LowCardinality` in `PREWHERE`. Fixes [#23515](https://github.com/ClickHouse/ClickHouse/issues/23515). [#27298](https://github.com/ClickHouse/ClickHouse/pull/27298) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `Cannot find column` error for queries with sampling. Was introduced in [#24574](https://github.com/ClickHouse/ClickHouse/issues/24574). Fixes [#26522](https://github.com/ClickHouse/ClickHouse/issues/26522). [#27301](https://github.com/ClickHouse/ClickHouse/pull/27301) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix Mysql protocol when using parallel formats (CSV / TSV). [#27326](https://github.com/ClickHouse/ClickHouse/pull/27326) ([Raúl Marín](https://github.com/Algunenano)). +* Fixed incorrect validation of partition id for MergeTree tables that created with old syntax. [#27328](https://github.com/ClickHouse/ClickHouse/pull/27328) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix incorrect result for query with row-level security, prewhere and LowCardinality filter. Fixes [#27179](https://github.com/ClickHouse/ClickHouse/issues/27179). [#27329](https://github.com/ClickHouse/ClickHouse/pull/27329) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* /proc/info contains metrics like. [#27361](https://github.com/ClickHouse/ClickHouse/pull/27361) ([Mike Kot](https://github.com/myrrc)). +* Fix distributed queries with zero shards and aggregation. [#27427](https://github.com/ClickHouse/ClickHouse/pull/27427) ([Azat Khuzhin](https://github.com/azat)). +* fix metric BackgroundMessageBrokerSchedulePoolTask, maybe mistyped。. [#27452](https://github.com/ClickHouse/ClickHouse/pull/27452) ([Ben](https://github.com/benbiti)). +* Fix crash during projection materialization when some parts contain missing columns. This fixes [#27512](https://github.com/ClickHouse/ClickHouse/issues/27512). [#27528](https://github.com/ClickHouse/ClickHouse/pull/27528) ([Amos Bird](https://github.com/amosbird)). +* Fixed underflow of the time value when constructing it from components. Closes [#27193](https://github.com/ClickHouse/ClickHouse/issues/27193). [#27605](https://github.com/ClickHouse/ClickHouse/pull/27605) ([Vasily Nemkov](https://github.com/Enmk)). +* After setting `max_memory_usage*` to non-zero value it was not possible to reset it back to 0 (unlimited). It's fixed. [#27638](https://github.com/ClickHouse/ClickHouse/pull/27638) ([Alexander Tokmakov](https://github.com/tavplubix)). +* - Fix bug with aliased column in `Distributed` table. [#27652](https://github.com/ClickHouse/ClickHouse/pull/27652) ([Vladimir C](https://github.com/vdimir)). +* Fixed another case of `Unexpected merged part ... intersecting drop range ...` error. [#27656](https://github.com/ClickHouse/ClickHouse/pull/27656) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix postgresql table function resulting in non-closing connections. Closes [#26088](https://github.com/ClickHouse/ClickHouse/issues/26088). [#27662](https://github.com/ClickHouse/ClickHouse/pull/27662) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix bad type cast when functions like `arrayHas` are applied to arrays of LowCardinality of Nullable of different non-numeric types like `DateTime` and `DateTime64`. In previous versions bad cast occurs. In new version it will lead to exception. This closes [#26330](https://github.com/ClickHouse/ClickHouse/issues/26330). [#27682](https://github.com/ClickHouse/ClickHouse/pull/27682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix column filtering with union distinct in subquery. Closes [#27578](https://github.com/ClickHouse/ClickHouse/issues/27578). [#27689](https://github.com/ClickHouse/ClickHouse/pull/27689) ([Kseniia Sumarokova](https://github.com/kssenii)). +* After https://github.com/ClickHouse/ClickHouse/pull/26384. To execute `GRANT WITH REPLACE OPTION` now the current user should have `GRANT OPTION` for access rights it's going to grant AND for access rights it's going to revoke. [#27701](https://github.com/ClickHouse/ClickHouse/pull/27701) ([Vitaly Baranov](https://github.com/vitlibar)). +* After https://github.com/ClickHouse/ClickHouse/pull/25687. Add backquotes for the default database shown in CREATE USER. [#27702](https://github.com/ClickHouse/ClickHouse/pull/27702) ([Vitaly Baranov](https://github.com/vitlibar)). +* Remove duplicated source files in CMakeLists.txt in arrow-cmake. [#27736](https://github.com/ClickHouse/ClickHouse/pull/27736) ([李扬](https://github.com/taiyang-li)). +* Fix possible crash when asynchronous connection draining is enabled and hedged connection is disabled. [#27774](https://github.com/ClickHouse/ClickHouse/pull/27774) ([Amos Bird](https://github.com/amosbird)). +* Prevent crashes for some formats when NULL (tombstone) message was coming from Kafka. Closes [#19255](https://github.com/ClickHouse/ClickHouse/issues/19255). [#27794](https://github.com/ClickHouse/ClickHouse/pull/27794) ([filimonov](https://github.com/filimonov)). +* Fix a rare bug in `DROP PART` which can lead to the error `Unexpected merged part intersects drop range`. [#27807](https://github.com/ClickHouse/ClickHouse/pull/27807) ([alesapin](https://github.com/alesapin)). +* Fix a couple of bugs that may cause replicas to diverge. [#27808](https://github.com/ClickHouse/ClickHouse/pull/27808) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### Build/Testing/Packaging Improvement +* Update RocksDB to 2021-07-16 master. [#26411](https://github.com/ClickHouse/ClickHouse/pull/26411) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* `clickhouse-test` supports SQL tests with [Jinja2](https://jinja.palletsprojects.com/en/3.0.x/templates/#synopsis) templates. [#26579](https://github.com/ClickHouse/ClickHouse/pull/26579) ([Vladimir C](https://github.com/vdimir)). +* Fix /clickhouse/window functions/tests/non distributed/errors/error window function in join. [#26744](https://github.com/ClickHouse/ClickHouse/pull/26744) ([vzakaznikov](https://github.com/vzakaznikov)). +* Enabling RBAC TestFlows tests and crossing out new fails. [#26747](https://github.com/ClickHouse/ClickHouse/pull/26747) ([vzakaznikov](https://github.com/vzakaznikov)). +* Tests: Fix CLICKHOUSE_CLIENT_SECURE with the default config. [#26901](https://github.com/ClickHouse/ClickHouse/pull/26901) ([Raúl Marín](https://github.com/Algunenano)). +* Fix linking of auxiliar programs when using dynamic libraries. [#26958](https://github.com/ClickHouse/ClickHouse/pull/26958) ([Raúl Marín](https://github.com/Algunenano)). +* Add CMake options to build with or without specific CPU instruction set. This is for [#17469](https://github.com/ClickHouse/ClickHouse/issues/17469) and [#27509](https://github.com/ClickHouse/ClickHouse/issues/27509). [#27508](https://github.com/ClickHouse/ClickHouse/pull/27508) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add support for build with `clang-13`. This closes [#27705](https://github.com/ClickHouse/ClickHouse/issues/27705). [#27714](https://github.com/ClickHouse/ClickHouse/pull/27714) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve support for build with `clang-13`. [#27777](https://github.com/ClickHouse/ClickHouse/pull/27777) ([Sergei Semin](https://github.com/syominsergey)). + +#### Other +* Rename `MaterializeMySQL` to `MaterializedMySQL`. [#26822](https://github.com/ClickHouse/ClickHouse/pull/26822) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Modify code comments'. [#26265](https://github.com/ClickHouse/ClickHouse/pull/26265) ([xiedeyantu](https://github.com/xiedeyantu)). +* NO CL ENTRY: 'Revert "Datatype Date32, support range 1925 to 2283"'. [#26352](https://github.com/ClickHouse/ClickHouse/pull/26352) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Fix CURR_DATABASE empty for 01034_move_partition_from_table_zookeeper.sh'. [#27164](https://github.com/ClickHouse/ClickHouse/pull/27164) ([小路](https://github.com/nicelulu)). +* NO CL ENTRY: 'DOCSUP-12413: macros support in functions cluster and clusterAllReplicas'. [#27759](https://github.com/ClickHouse/ClickHouse/pull/27759) ([olgarev](https://github.com/olgarev)). +* NO CL ENTRY: 'Revert "less sys calls #2: make vdso work again"'. [#27829](https://github.com/ClickHouse/ClickHouse/pull/27829) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Do not miss exceptions from the ThreadPool"'. [#27844](https://github.com/ClickHouse/ClickHouse/pull/27844) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.9.2.17-stable.md b/docs/changelogs/v21.9.2.17-stable.md new file mode 100644 index 00000000000..3f132b983c0 --- /dev/null +++ b/docs/changelogs/v21.9.2.17-stable.md @@ -0,0 +1,45 @@ +### ClickHouse release v21.9.2.17-stable FIXME as compared to v21.9.1.8000-prestable + +#### Improvement +* Backported in [#27894](https://github.com/ClickHouse/ClickHouse/issues/27894): Allow symlinks for library dictionaty path. [#27815](https://github.com/ClickHouse/ClickHouse/pull/27815) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#28153](https://github.com/ClickHouse/ClickHouse/issues/28153): Use Multipart copy upload for large S3 objects. [#27858](https://github.com/ClickHouse/ClickHouse/pull/27858) ([ianton-ru](https://github.com/ianton-ru)). +* Backported in [#28643](https://github.com/ClickHouse/ClickHouse/issues/28643): Fix strange sessions expiration logic in Keeper. Probably it should help in CI: https://clickhouse-test-reports.s3.yandex.net/0/6bd9b82141c98dcd7796fd9d08326831095ba519/stress_test_(debug).html#fail1. [#28519](https://github.com/ClickHouse/ClickHouse/pull/28519) ([alesapin](https://github.com/alesapin)). + +#### Bug Fix +* Backported in [#27981](https://github.com/ClickHouse/ClickHouse/issues/27981): Bugfix for windowFunnel's "strict" mode. This fixes [#27469](https://github.com/ClickHouse/ClickHouse/issues/27469). [#27563](https://github.com/ClickHouse/ClickHouse/pull/27563) ([achimbab](https://github.com/achimbab)). +* Backported in [#27922](https://github.com/ClickHouse/ClickHouse/issues/27922): After https://github.com/ClickHouse/ClickHouse/pull/26864. Fix shutdown of `NamedSessionStorage`: session contexts stored in `NamedSessionStorage` are now destroyed before destroying the global context. [#27875](https://github.com/ClickHouse/ClickHouse/pull/27875) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#27926](https://github.com/ClickHouse/ClickHouse/issues/27926): Fix PostgreSQL-style cast (`::` operator) with negative numbers. [#27876](https://github.com/ClickHouse/ClickHouse/pull/27876) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#27959](https://github.com/ClickHouse/ClickHouse/issues/27959): Fix selecting with extremes from a column of the type `LowCardinality(UUID)`. [#27918](https://github.com/ClickHouse/ClickHouse/pull/27918) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#27954](https://github.com/ClickHouse/ClickHouse/issues/27954): Check cluster name before creating Distributed table, do not allow to create a table with incorrect cluster name. Fixes [#27832](https://github.com/ClickHouse/ClickHouse/issues/27832). [#27927](https://github.com/ClickHouse/ClickHouse/pull/27927) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#28000](https://github.com/ClickHouse/ClickHouse/issues/28000): Fix checking access grants when executing GRANT WITH REPLACE statement with ON CLUSTER clause. This PR improves fix https://github.com/ClickHouse/ClickHouse/pull/27701. [#27983](https://github.com/ClickHouse/ClickHouse/pull/27983) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#28210](https://github.com/ClickHouse/ClickHouse/issues/28210): Fix cases, when read buffer fails with 'attempt to read after end of file'. Closes [#26149](https://github.com/ClickHouse/ClickHouse/issues/26149). [#28150](https://github.com/ClickHouse/ClickHouse/pull/28150) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Backported in [#28750](https://github.com/ClickHouse/ClickHouse/issues/28750): Fix transformation of disjunctions chain to `IN` (controlled by settings `optimize_min_equality_disjunction_chain_length`) in distributed queries with settings `legacy_column_name_of_tuple_literal = 0`. [#28658](https://github.com/ClickHouse/ClickHouse/pull/28658) ([Anton Popov](https://github.com/CurtizJ)). + +#### Build/Testing/Packaging Improvement +* Backported in [#28029](https://github.com/ClickHouse/ClickHouse/issues/28029): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#27973](https://github.com/ClickHouse/ClickHouse/issues/27973): Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#28118](https://github.com/ClickHouse/ClickHouse/issues/28118): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). +* Backported in [#28182](https://github.com/ClickHouse/ClickHouse/issues/28182): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). +* Backported in [#28260](https://github.com/ClickHouse/ClickHouse/issues/28260): Multiple small fixes for projections. See detailed description in pr. [#28178](https://github.com/ClickHouse/ClickHouse/pull/28178) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#28257](https://github.com/ClickHouse/ClickHouse/issues/28257): Fix incorrect behavior in `clickhouse-keeper` when list watches (`getChildren`) triggered with `set` requests for children. [#28190](https://github.com/ClickHouse/ClickHouse/pull/28190) ([alesapin](https://github.com/alesapin)). +* Backported in [#28343](https://github.com/ClickHouse/ClickHouse/issues/28343): Fix a rare bug in `clickhouse-keeper` when the client can receive a watch response before request-response. [#28197](https://github.com/ClickHouse/ClickHouse/pull/28197) ([alesapin](https://github.com/alesapin)). +* Backported in [#28261](https://github.com/ClickHouse/ClickHouse/issues/28261): Fix possible read of uninitialized memory for queries with `Nullable(LowCardinality)` type and extremes. Fixes [#28165](https://github.com/ClickHouse/ClickHouse/issues/28165). [#28205](https://github.com/ClickHouse/ClickHouse/pull/28205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#28253](https://github.com/ClickHouse/ClickHouse/issues/28253): Fix reading of custom TLD w/o new line at EOF. [#28213](https://github.com/ClickHouse/ClickHouse/pull/28213) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#28294](https://github.com/ClickHouse/ClickHouse/issues/28294): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28401](https://github.com/ClickHouse/ClickHouse/issues/28401): Fix intersecting parts due to new part had been replaced with an empty part. [#28310](https://github.com/ClickHouse/ClickHouse/pull/28310) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#28688](https://github.com/ClickHouse/ClickHouse/issues/28688): Fix NOT-IN index optimization when not all key columns are used. This fixes [#28120](https://github.com/ClickHouse/ClickHouse/issues/28120). [#28315](https://github.com/ClickHouse/ClickHouse/pull/28315) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#28435](https://github.com/ClickHouse/ClickHouse/issues/28435): Fix non joined rows from nullable column. Close [#27691](https://github.com/ClickHouse/ClickHouse/issues/27691). [#28349](https://github.com/ClickHouse/ClickHouse/pull/28349) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#28645](https://github.com/ClickHouse/ClickHouse/issues/28645): Fix rare case when changes of `clickhouse-keeper` settings may lead to lost logs and server hung. [#28360](https://github.com/ClickHouse/ClickHouse/pull/28360) ([alesapin](https://github.com/alesapin)). +* Backported in [#28507](https://github.com/ClickHouse/ClickHouse/issues/28507): Fix lack of quotes for table names in MaterializedPostgreSQL engine. Closes [#28316](https://github.com/ClickHouse/ClickHouse/issues/28316). [#28433](https://github.com/ClickHouse/ClickHouse/pull/28433) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#28480](https://github.com/ClickHouse/ClickHouse/issues/28480): Fixed possible ZooKeeper watches leak on background processing of distributed DDL queue. Closes [#26036](https://github.com/ClickHouse/ClickHouse/issues/26036). [#28446](https://github.com/ClickHouse/ClickHouse/pull/28446) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#28572](https://github.com/ClickHouse/ClickHouse/issues/28572): Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)). +* Backported in [#28594](https://github.com/ClickHouse/ClickHouse/issues/28594): Fix `There is no subcolumn` error, while select from tables, which have `Nested` columns and scalar columns with dot in name and the same prefix as `Nested` (e.g. `n.id UInt32, n.arr1 Array(UInt64), n.arr2 Array(UInt64)`). [#28531](https://github.com/ClickHouse/ClickHouse/pull/28531) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28657](https://github.com/ClickHouse/ClickHouse/issues/28657): Fix UUID overlap in DROP TABLE for internal DDL from MaterializeMySQL. [#28533](https://github.com/ClickHouse/ClickHouse/pull/28533) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#28568](https://github.com/ClickHouse/ClickHouse/issues/28568): Fix endless loop for truncated bzip2 archive. [#28543](https://github.com/ClickHouse/ClickHouse/pull/28543) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#28704](https://github.com/ClickHouse/ClickHouse/issues/28704): - Fix the number of arguments required by s2RectAdd and s2RectContains functions. [#28663](https://github.com/ClickHouse/ClickHouse/pull/28663) ([Bharat Nallan](https://github.com/bharatnc)). +* Backported in [#28715](https://github.com/ClickHouse/ClickHouse/issues/28715): Add Settings.Names, Settings.Values aliases for system.processes table. [#28685](https://github.com/ClickHouse/ClickHouse/pull/28685) ([Vitaly Orlov](https://github.com/orloffv)). +* Backported in [#28744](https://github.com/ClickHouse/ClickHouse/issues/28744): Fix the coredump in the creation of distributed tables, when the parameters passed in are wrong. [#28686](https://github.com/ClickHouse/ClickHouse/pull/28686) ([Zhiyong Wang](https://github.com/ljcui)). + diff --git a/docs/changelogs/v21.9.3.30-stable.md b/docs/changelogs/v21.9.3.30-stable.md new file mode 100644 index 00000000000..3b665365668 --- /dev/null +++ b/docs/changelogs/v21.9.3.30-stable.md @@ -0,0 +1,16 @@ +### ClickHouse release v21.9.3.30-stable FIXME as compared to v21.9.2.17-stable + +#### Improvement +* Backported in [#28897](https://github.com/ClickHouse/ClickHouse/issues/28897): Use real tmp file instead of predefined "rows_sources" for vertical merges. This avoids generating garbage directories in tmp disks. [#28299](https://github.com/ClickHouse/ClickHouse/pull/28299) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#28815](https://github.com/ClickHouse/ClickHouse/issues/28815): Fix possible crash for `SELECT` with partially created aggregate projection in case of exception. [#28700](https://github.com/ClickHouse/ClickHouse/pull/28700) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#28789](https://github.com/ClickHouse/ClickHouse/issues/28789): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)). +* Backported in [#28842](https://github.com/ClickHouse/ClickHouse/issues/28842): Fix expressions compilation with short circuit evaluation. [#28821](https://github.com/ClickHouse/ClickHouse/pull/28821) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#28993](https://github.com/ClickHouse/ClickHouse/issues/28993): Fixed a race condition between `DROP PART` and `REPLACE/MOVE PARTITION` that might cause replicas to diverge in rare cases. [#28864](https://github.com/ClickHouse/ClickHouse/pull/28864) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#28949](https://github.com/ClickHouse/ClickHouse/issues/28949): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#28925](https://github.com/ClickHouse/ClickHouse/issues/28925): Fix bug with LowCardinality in short-curcuit function evaluation. Closes [#28884](https://github.com/ClickHouse/ClickHouse/issues/28884). [#28887](https://github.com/ClickHouse/ClickHouse/pull/28887) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#28927](https://github.com/ClickHouse/ClickHouse/issues/28927): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29025](https://github.com/ClickHouse/ClickHouse/issues/29025): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.9.4.35-stable.md b/docs/changelogs/v21.9.4.35-stable.md new file mode 100644 index 00000000000..8b919ecb268 --- /dev/null +++ b/docs/changelogs/v21.9.4.35-stable.md @@ -0,0 +1,6 @@ +### ClickHouse release v21.9.4.35-stable FIXME as compared to v21.9.3.30-stable + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#29191](https://github.com/ClickHouse/ClickHouse/issues/29191): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.9.5.16-stable.md b/docs/changelogs/v21.9.5.16-stable.md new file mode 100644 index 00000000000..7287c58064d --- /dev/null +++ b/docs/changelogs/v21.9.5.16-stable.md @@ -0,0 +1,48 @@ +### ClickHouse release v21.9.5.16-stable FIXME as compared to v21.9.4.35-stable + +#### Improvement +* Backported in [#29897](https://github.com/ClickHouse/ClickHouse/issues/29897): Added partitioned table prefix 'p' for the query for fetching replica identity index. [#29828](https://github.com/ClickHouse/ClickHouse/pull/29828) ([Shoh Jahon](https://github.com/Shohjahon)). +* Backported in [#29943](https://github.com/ClickHouse/ClickHouse/issues/29943): Update zoneinfo files to 2021c. [#29925](https://github.com/ClickHouse/ClickHouse/pull/29925) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix +* Backported in [#29775](https://github.com/ClickHouse/ClickHouse/issues/29775): Allow using a materialized column as the sharding key in a distributed table even if `insert_allow_materialized_columns=0`:. [#28637](https://github.com/ClickHouse/ClickHouse/pull/28637) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#29126](https://github.com/ClickHouse/ClickHouse/issues/29126): Fix bug in `clickhouse-keeper-converter` which can lead to incorrect ZooKeeper log deserialization. [#29071](https://github.com/ClickHouse/ClickHouse/pull/29071) ([小路](https://github.com/nicelulu)). +* Backported in [#29972](https://github.com/ClickHouse/ClickHouse/issues/29972): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#30052](https://github.com/ClickHouse/ClickHouse/issues/30052): Fix releasing query ID and session ID at the end of query processing while handing gRPC call. This PR fixes flaky test [test_grpc_protocol/test.py::test_session](https://clickhouse-test-reports.s3.yandex.net/0/1ac03811a2df9717fa7c633d1af03def821d24b6/integration_tests_(memory).html). [#29954](https://github.com/ClickHouse/ClickHouse/pull/29954) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#29055](https://github.com/ClickHouse/ClickHouse/issues/29055): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#29107](https://github.com/ClickHouse/ClickHouse/issues/29107): Fix waiting for mutation with `mutations_sync=2`. [#28889](https://github.com/ClickHouse/ClickHouse/pull/28889) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29815](https://github.com/ClickHouse/ClickHouse/issues/29815): Do not allow to reuse previous credentials in case of inter-server secret (Before INSERT via Buffer/Kafka to Distributed table with interserver secret configured for that cluster, may re-use previously set user for that connection). [#29060](https://github.com/ClickHouse/ClickHouse/pull/29060) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29399](https://github.com/ClickHouse/ClickHouse/issues/29399): Send normal `Database doesn't exist error` (`UNKNOWN_DATABASE`) to the client (via TCP) instead of `Attempt to read after eof` (`ATTEMPT_TO_READ_AFTER_EOF`). [#29229](https://github.com/ClickHouse/ClickHouse/pull/29229) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29356](https://github.com/ClickHouse/ClickHouse/issues/29356): Fix possible `Table columns structure in ZooKeeper is different from local table structure` exception while recreating or creating new replicas of `ReplicatedMergeTree`, when one of table columns have default expressions with case-insensitive functions. [#29266](https://github.com/ClickHouse/ClickHouse/pull/29266) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#29451](https://github.com/ClickHouse/ClickHouse/issues/29451): Fix failed assertion in ReadBufferFromHDFS. Update libhdfs3 library to be able to run in tests in debug. Closes [#29251](https://github.com/ClickHouse/ClickHouse/issues/29251). Closes [#27814](https://github.com/ClickHouse/ClickHouse/issues/27814). [#29276](https://github.com/ClickHouse/ClickHouse/pull/29276) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29301](https://github.com/ClickHouse/ClickHouse/issues/29301): Fix connection timeouts (`send_timeout`/`receive_timeout`). [#29282](https://github.com/ClickHouse/ClickHouse/pull/29282) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29383](https://github.com/ClickHouse/ClickHouse/issues/29383): Remove window function `nth_value` as it is not memory-safe. This closes [#29347](https://github.com/ClickHouse/ClickHouse/issues/29347). [#29348](https://github.com/ClickHouse/ClickHouse/pull/29348) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#29438](https://github.com/ClickHouse/ClickHouse/issues/29438): Fix replicated access storage not shutting down cleanly when misconfigured. [#29388](https://github.com/ClickHouse/ClickHouse/pull/29388) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Backported in [#29489](https://github.com/ClickHouse/ClickHouse/issues/29489): Fix Logical error `Cannot capture columns` in functions greatest/least. Closes [#29334](https://github.com/ClickHouse/ClickHouse/issues/29334). [#29454](https://github.com/ClickHouse/ClickHouse/pull/29454) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#29536](https://github.com/ClickHouse/ClickHouse/issues/29536): Fix possible `Block structure mismatch` for subqueries with pushed-down `HAVING` predicate. Fixes [#29010](https://github.com/ClickHouse/ClickHouse/issues/29010). [#29475](https://github.com/ClickHouse/ClickHouse/pull/29475) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29592](https://github.com/ClickHouse/ClickHouse/issues/29592): In ODBC bridge add retries for error Invalid cursor state. It is a retriable error. Closes [#29473](https://github.com/ClickHouse/ClickHouse/issues/29473). [#29518](https://github.com/ClickHouse/ClickHouse/pull/29518) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29572](https://github.com/ClickHouse/ClickHouse/issues/29572): Fix bug in check `pathStartsWith` becuase there was bug with the usage of `std::mismatch`: ` The behavior is undefined if the second range is shorter than the first range.`. [#29531](https://github.com/ClickHouse/ClickHouse/pull/29531) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#29630](https://github.com/ClickHouse/ClickHouse/issues/29630): Fix rare segfault in `ALTER MODIFY` query when using incorrect table identifier in `DEFAULT` expression like `x.y.z...` Fixes [#29184](https://github.com/ClickHouse/ClickHouse/issues/29184). [#29573](https://github.com/ClickHouse/ClickHouse/pull/29573) ([alesapin](https://github.com/alesapin)). +* Backported in [#29658](https://github.com/ClickHouse/ClickHouse/issues/29658): Fix JIT expression compilation with aliases and short-circuit expression evaluation. Closes [#29403](https://github.com/ClickHouse/ClickHouse/issues/29403). [#29574](https://github.com/ClickHouse/ClickHouse/pull/29574) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#29751](https://github.com/ClickHouse/ClickHouse/issues/29751): Condition in filter predicate could be lost after push-down optimisation. [#29625](https://github.com/ClickHouse/ClickHouse/pull/29625) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#29849](https://github.com/ClickHouse/ClickHouse/issues/29849): Fix concurrent access to `LowCardinality` during `GROUP BY` (leads to SIGSEGV). [#29782](https://github.com/ClickHouse/ClickHouse/pull/29782) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#29909](https://github.com/ClickHouse/ClickHouse/issues/29909): Fix bad cast in `ATTACH TABLE ... FROM 'path'` query when non-string literal is used instead of path. It may lead to reading of uninitialized memory. [#29790](https://github.com/ClickHouse/ClickHouse/pull/29790) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#29865](https://github.com/ClickHouse/ClickHouse/issues/29865): Avoid `Timeout exceeded: elapsed 18446744073.709553 seconds` error that might happen in extremely rare cases, presumably due to some bug in kernel. Fixes [#29154](https://github.com/ClickHouse/ClickHouse/issues/29154). [#29811](https://github.com/ClickHouse/ClickHouse/pull/29811) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30024](https://github.com/ClickHouse/ClickHouse/issues/30024): MaterializedMySQL: Fix an issue where if the connection to MySQL was lost, only parts of a transaction could be processed. [#29837](https://github.com/ClickHouse/ClickHouse/pull/29837) ([Håvard Kvålen](https://github.com/havardk)). +* Backported in [#29878](https://github.com/ClickHouse/ClickHouse/issues/29878): Fix system tables recreation check (fails to detect changes in enum values). [#29857](https://github.com/ClickHouse/ClickHouse/pull/29857) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30057](https://github.com/ClickHouse/ClickHouse/issues/30057): Fix potential resource leak of the concurrent query limit of merge tree tables introduced in https://github.com/ClickHouse/ClickHouse/pull/19544 . [#29879](https://github.com/ClickHouse/ClickHouse/pull/29879) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#30204](https://github.com/ClickHouse/ClickHouse/issues/30204): Fix data-race between `LogSink::writeMarks()` and `LogSource` in `StorageLog`. [#29946](https://github.com/ClickHouse/ClickHouse/pull/29946) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30209](https://github.com/ClickHouse/ClickHouse/issues/30209): Fix possible data-race between `FileChecker` and `StorageLog`/`StorageStripeLog`. [#29959](https://github.com/ClickHouse/ClickHouse/pull/29959) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30070](https://github.com/ClickHouse/ClickHouse/issues/30070): Fix crash of sample by `tuple()`, closes [#30004](https://github.com/ClickHouse/ClickHouse/issues/30004). [#30016](https://github.com/ClickHouse/ClickHouse/pull/30016) ([flynn](https://github.com/ucasfl)). +* Backported in [#30128](https://github.com/ClickHouse/ClickHouse/issues/30128): Dropped `Memory` database might reappear after server restart, it's fixed ([#29795](https://github.com/ClickHouse/ClickHouse/issues/29795)). Also added `force_remove_data_recursively_on_drop` setting as a workaround for `Directory not empty` error when dropping `Ordinary` database (because it's not possible to remove data leftovers manually in cloud environment). [#30054](https://github.com/ClickHouse/ClickHouse/pull/30054) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30262](https://github.com/ClickHouse/ClickHouse/issues/30262): FlatDictionary, HashedDictionary fix bytes_allocated calculation for nullable attributes. [#30238](https://github.com/ClickHouse/ClickHouse/pull/30238) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#30306](https://github.com/ClickHouse/ClickHouse/issues/30306): Fix crash with shortcircuit and lowcardinality in multiIf. [#30243](https://github.com/ClickHouse/ClickHouse/pull/30243) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#30290](https://github.com/ClickHouse/ClickHouse/issues/30290): Fix ComplexKeyHashedDictionary, ComplexKeySparseHashedDictionary parsing `preallocate` option from layout config. [#30246](https://github.com/ClickHouse/ClickHouse/pull/30246) ([Maksim Kita](https://github.com/kitaisreal)). + +#### NO CL CATEGORY + +* Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30185](https://github.com/ClickHouse/ClickHouse/pull/30185) ([Raúl Marín](https://github.com/Algunenano)). + diff --git a/docs/changelogs/v21.9.6.24-stable.md b/docs/changelogs/v21.9.6.24-stable.md new file mode 100644 index 00000000000..f1d097ab646 --- /dev/null +++ b/docs/changelogs/v21.9.6.24-stable.md @@ -0,0 +1,57 @@ +### ClickHouse release v21.9.6.24-stable FIXME as compared to v21.9.5.16-stable + +#### New Feature +* Backported in [#30714](https://github.com/ClickHouse/ClickHouse/issues/30714): CompiledExpressionCache limit elements size using `compiled_expression_cache_elements_size` setting. [#30667](https://github.com/ClickHouse/ClickHouse/pull/30667) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Performance Improvement +* Backported in [#31734](https://github.com/ClickHouse/ClickHouse/issues/31734): Improve performance of JSON and XML output formats. [#31673](https://github.com/ClickHouse/ClickHouse/pull/31673) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Improvement +* Backported in [#30470](https://github.com/ClickHouse/ClickHouse/issues/30470): Allow symlinks to files in user_files directory for file table function. [#30309](https://github.com/ClickHouse/ClickHouse/pull/30309) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Bug Fix +* Backported in [#30664](https://github.com/ClickHouse/ClickHouse/issues/30664): Fix reading from empty file on encrypted disk. [#30494](https://github.com/ClickHouse/ClickHouse/pull/30494) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#31371](https://github.com/ClickHouse/ClickHouse/issues/31371): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). +* Backported in [#31576](https://github.com/ClickHouse/ClickHouse/issues/31576): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release + +* Backported in [#30916](https://github.com/ClickHouse/ClickHouse/issues/30916): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#30823](https://github.com/ClickHouse/ClickHouse/issues/30823): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30609](https://github.com/ClickHouse/ClickHouse/issues/30609): Fix bad optimizations of ORDER BY if it contains WITH FILL. This closes [#28908](https://github.com/ClickHouse/ClickHouse/issues/28908). This closes [#26049](https://github.com/ClickHouse/ClickHouse/issues/26049). [#28910](https://github.com/ClickHouse/ClickHouse/pull/28910) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#30765](https://github.com/ClickHouse/ClickHouse/issues/30765): Fix hanging DDL queries on Replicated database while adding a new replica. [#29328](https://github.com/ClickHouse/ClickHouse/pull/29328) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Backported in [#30505](https://github.com/ClickHouse/ClickHouse/issues/30505): Fixed incorrect behaviour of setting `materialized_postgresql_tables_list` at server restart. Found in [#28529](https://github.com/ClickHouse/ClickHouse/issues/28529). [#29686](https://github.com/ClickHouse/ClickHouse/pull/29686) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#30464](https://github.com/ClickHouse/ClickHouse/issues/30464): Support nullable arguments in function `initializeAggregation`. [#30177](https://github.com/ClickHouse/ClickHouse/pull/30177) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#30657](https://github.com/ClickHouse/ClickHouse/issues/30657): Fix `[I]LIKE` function. Closes [#28661](https://github.com/ClickHouse/ClickHouse/issues/28661). [#30244](https://github.com/ClickHouse/ClickHouse/pull/30244) ([Nikolay Degterinsky](https://github.com/evillique)). +* Backported in [#30524](https://github.com/ClickHouse/ClickHouse/issues/30524): Fixed segfault which might happen if session expired during execution of REPLACE PARTITION. [#30432](https://github.com/ClickHouse/ClickHouse/pull/30432) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30586](https://github.com/ClickHouse/ClickHouse/issues/30586): * Fix deadlock on ALTER with scalar subquery to the same table, close [#30461](https://github.com/ClickHouse/ClickHouse/issues/30461). [#30492](https://github.com/ClickHouse/ClickHouse/pull/30492) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#30606](https://github.com/ClickHouse/ClickHouse/issues/30606): Limit push down optimization could cause a error `Cannot find column`. Fixes [#30438](https://github.com/ClickHouse/ClickHouse/issues/30438). [#30562](https://github.com/ClickHouse/ClickHouse/pull/30562) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Backported in [#30752](https://github.com/ClickHouse/ClickHouse/issues/30752): Functions for case-insensitive search in UTF8 strings like `positionCaseInsensitiveUTF8` and `countSubstringsCaseInsensitiveUTF8` might find substrings that actually does not match, it's fixed. [#30663](https://github.com/ClickHouse/ClickHouse/pull/30663) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30711](https://github.com/ClickHouse/ClickHouse/issues/30711): Fix PREWHERE with WHERE in case of always true PREWHERE. [#30668](https://github.com/ClickHouse/ClickHouse/pull/30668) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#30769](https://github.com/ClickHouse/ClickHouse/issues/30769): Fixed a race condition between `REPLACE/MOVE PARTITION` and background merge in non-replicated `MergeTree` that might cause a part of moved/replaced data to remain in partition. Fixes [#29327](https://github.com/ClickHouse/ClickHouse/issues/29327). [#30717](https://github.com/ClickHouse/ClickHouse/pull/30717) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30858](https://github.com/ClickHouse/ClickHouse/issues/30858): Fixed ambiguity when extracting auxiliary ZooKeeper name from ZooKeeper path in `ReplicatedMergeTree`. Previously server might fail to start with `Unknown auxiliary ZooKeeper name` if ZooKeeper path contains a colon. Fixes [#29052](https://github.com/ClickHouse/ClickHouse/issues/29052). Also it was allowed to specify ZooKeeper path that does not start with slash, but now it's deprecated and creation of new tables with such path is not allowed. Slashes and colons in auxiliary ZooKeeper names are not allowed too. [#30822](https://github.com/ClickHouse/ClickHouse/pull/30822) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#30924](https://github.com/ClickHouse/ClickHouse/issues/30924): Fix set index not used in AND/OR expressions when there are more than two operands. This fixes [#30416](https://github.com/ClickHouse/ClickHouse/issues/30416) . [#30887](https://github.com/ClickHouse/ClickHouse/pull/30887) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#31290](https://github.com/ClickHouse/ClickHouse/issues/31290): Fix some corner cases with intersect/except. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31153](https://github.com/ClickHouse/ClickHouse/issues/31153): Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)). +* Backported in [#31039](https://github.com/ClickHouse/ClickHouse/issues/31039): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31131](https://github.com/ClickHouse/ClickHouse/issues/31131): Fix JSONValue/Query with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31205](https://github.com/ClickHouse/ClickHouse/issues/31205): Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31375](https://github.com/ClickHouse/ClickHouse/issues/31375): Fix StorageMerge with aliases and where (it did not work before at all). Closes [#28802](https://github.com/ClickHouse/ClickHouse/issues/28802). [#31044](https://github.com/ClickHouse/ClickHouse/pull/31044) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#31254](https://github.com/ClickHouse/ClickHouse/issues/31254): Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)). +* Backported in [#31520](https://github.com/ClickHouse/ClickHouse/issues/31520): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)). +* Backported in [#31553](https://github.com/ClickHouse/ClickHouse/issues/31553): Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Backported in [#31583](https://github.com/ClickHouse/ClickHouse/issues/31583): * Disable `partial_merge_join_left_table_buffer_bytes` before bug in this optimization is fixed. See [#31009](https://github.com/ClickHouse/ClickHouse/issues/31009)). * Remove redundant option `partial_merge_join_optimizations`. [#31528](https://github.com/ClickHouse/ClickHouse/pull/31528) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#31601](https://github.com/ClickHouse/ClickHouse/issues/31601): Fix invalid generated JSON when only column names contain invalid UTF-8 sequences. [#31534](https://github.com/ClickHouse/ClickHouse/pull/31534) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Backported in [#31746](https://github.com/ClickHouse/ClickHouse/issues/31746): `RENAME TABLE` query worked incorrectly on attempt to rename an DDL dictionary in `Ordinary` database, it's fixed. [#31638](https://github.com/ClickHouse/ClickHouse/pull/31638) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31793](https://github.com/ClickHouse/ClickHouse/issues/31793): Settings `input_format_allow_errors_num` and `input_format_allow_errors_ratio` did not work for parsing of domain types, such as `IPv4`, it's fixed. Fixes [#31686](https://github.com/ClickHouse/ClickHouse/issues/31686). [#31697](https://github.com/ClickHouse/ClickHouse/pull/31697) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31828](https://github.com/ClickHouse/ClickHouse/issues/31828): Fixed `there are no such cluster here` error on execution of `ON CLUSTER` query if specified cluster name is name of `Replicated` database. [#31723](https://github.com/ClickHouse/ClickHouse/pull/31723) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#31817](https://github.com/ClickHouse/ClickHouse/issues/31817): Fix race in JSONEachRowWithProgress output format when data and lines with progress are mixed in output. [#31736](https://github.com/ClickHouse/ClickHouse/pull/31736) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#31759](https://github.com/ClickHouse/ClickHouse/issues/31759): Fix usage of `Buffer` table engine with type `Map`. Fixes [#30546](https://github.com/ClickHouse/ClickHouse/issues/30546). [#31742](https://github.com/ClickHouse/ClickHouse/pull/31742) ([Anton Popov](https://github.com/CurtizJ)). +* Backported in [#31893](https://github.com/ClickHouse/ClickHouse/issues/31893): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#32030](https://github.com/ClickHouse/ClickHouse/issues/32030): Fix invalid cast of nullable type when nullable primary key is used. This fixes [#31075](https://github.com/ClickHouse/ClickHouse/issues/31075). [#31823](https://github.com/ClickHouse/ClickHouse/pull/31823) ([Amos Bird](https://github.com/amosbird)). +* Backported in [#32078](https://github.com/ClickHouse/ClickHouse/issues/32078): Fix a bug about function transform with decimal args. [#31839](https://github.com/ClickHouse/ClickHouse/pull/31839) ([Shuai li](https://github.com/loneylee)). +* Backported in [#31907](https://github.com/ClickHouse/ClickHouse/issues/31907): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v22.1.1.2542-prestable.md b/docs/changelogs/v22.1.1.2542-prestable.md index f6418c5c3b9..e1e30e28ec6 100644 --- a/docs/changelogs/v22.1.1.2542-prestable.md +++ b/docs/changelogs/v22.1.1.2542-prestable.md @@ -6,7 +6,7 @@ * Add `left`, `right`, `leftUTF8`, `rightUTF8` functions. Fix error in implementation of `substringUTF8` function with negative offset (offset from the end of string). The functions `left` and `right` were previously implemented in parser. Upgrade notes: distributed queries with `left` or `right` functions without aliases may throw exception if cluster contains different versions of clickhouse-server. If you are upgrading your cluster and encounter this error, you should finish upgrading your cluster to ensure all nodes have the same version. Also you can add aliases (`AS something`) to the columns in your queries to avoid this issue. [#33407](https://github.com/ClickHouse/ClickHouse/pull/33407) ([Alexey Milovidov](https://github.com/alexey-milovidov)). #### New Feature -* Implemented sparse serialization. It can reduce usage of disk space and improve performance of some queries for columns, which contain a lot of default (zero) values. It can be enabled by setting `ratio_for_sparse_serialization`. Sparse serialization will be chosen dynamically for column, if it has ratio of number of default values to number of all values above that threshold. Serialization (default or sparse) will be fixed for every column in part, but may varies between parts. [#22535](https://github.com/ClickHouse/ClickHouse/pull/22535) ([Anton Popov](https://github.com/CurtizJ)). +* Implemented sparse serialization. It can reduce usage of disk space and improve performance of some queries for columns, which contain a lot of default (zero) values. It can be enabled by setting `ratio_of_defaults_for_sparse_serialization`. Sparse serialization will be chosen dynamically for column, if it has ratio of number of default values to number of all values above that threshold. Serialization (default or sparse) will be fixed for every column in part, but may varies between parts. [#22535](https://github.com/ClickHouse/ClickHouse/pull/22535) ([Anton Popov](https://github.com/CurtizJ)). * add grouping sets function, like GROUP BY grouping sets (a, b, (a, b)). [#26869](https://github.com/ClickHouse/ClickHouse/pull/26869) ([taylor12805](https://github.com/taylor12805)). * Added an ability to read from all replicas within a shard during distributed query. To enable this, set `allow_experimental_parallel_reading_from_replicas=true` and `max_parallel_replicas` to any number. This closes [#26748](https://github.com/ClickHouse/ClickHouse/issues/26748). [#29279](https://github.com/ClickHouse/ClickHouse/pull/29279) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Start and stop servers when hosts and ports configuration changes. [#30549](https://github.com/ClickHouse/ClickHouse/pull/30549) ([Kevin Michel](https://github.com/kmichel-aiven)). diff --git a/docs/changelogs/v22.1.3.7-stable.md b/docs/changelogs/v22.1.3.7-stable.md index ffb0ec6048d..f8bb8821031 100644 --- a/docs/changelogs/v22.1.3.7-stable.md +++ b/docs/changelogs/v22.1.3.7-stable.md @@ -1,2 +1,9 @@ ### ClickHouse release v22.1.3.7-stable FIXME as compared to v22.1.2.2-stable +#### Improvement +* Backported in [#33793](https://github.com/ClickHouse/ClickHouse/issues/33793): Create parent directories in DiskS3::restoreFileOperations method. [#33730](https://github.com/ClickHouse/ClickHouse/pull/33730) ([ianton-ru](https://github.com/ianton-ru)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#33898](https://github.com/ClickHouse/ClickHouse/issues/33898): Fix usage of sparse columns (which can be enabled by experimental setting `ratio_of_defaults_for_sparse_serialization`). [#33849](https://github.com/ClickHouse/ClickHouse/pull/33849) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/en/development/adding_test_queries.md b/docs/en/development/adding_test_queries.md index 9b993a96ed5..ca47818dad7 100644 --- a/docs/en/development/adding_test_queries.md +++ b/docs/en/development/adding_test_queries.md @@ -106,7 +106,7 @@ vim tests/queries/0_stateless/01521_dummy_test.sql 4) run the test, and put the result of that into the reference file: ``` -clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee tests/queries/0_stateless/01521_dummy_test.reference +clickhouse-client -nm < tests/queries/0_stateless/01521_dummy_test.sql | tee tests/queries/0_stateless/01521_dummy_test.reference ``` 5) ensure everything is correct, if the test output is incorrect (due to some bug for example), adjust the reference file using text editor. diff --git a/docs/en/development/cmake-in-clickhouse.md b/docs/en/development/cmake-in-clickhouse.md index 65d280df902..a2ea99ecb67 100644 --- a/docs/en/development/cmake-in-clickhouse.md +++ b/docs/en/development/cmake-in-clickhouse.md @@ -13,11 +13,6 @@ cmake .. \ -DCMAKE_C_COMPILER=$(which clang-13) \ -DCMAKE_CXX_COMPILER=$(which clang++-13) \ -DCMAKE_BUILD_TYPE=Debug \ - -DENABLE_CLICKHOUSE_ALL=OFF \ - -DENABLE_CLICKHOUSE_SERVER=ON \ - -DENABLE_CLICKHOUSE_CLIENT=ON \ - -DENABLE_LIBRARIES=OFF \ - -DUSE_UNWIND=ON \ -DENABLE_UTILS=OFF \ -DENABLE_TESTS=OFF ``` diff --git a/docs/tools/deploy-to-test.sh b/docs/tools/deploy-to-test.sh index 30771052535..a7a922137d5 100755 --- a/docs/tools/deploy-to-test.sh +++ b/docs/tools/deploy-to-test.sh @@ -12,12 +12,11 @@ # set -ex -BASE_DIR=$(dirname $(readlink -f $0)) +BASE_DIR=$(dirname "$(readlink -f "$0")") GIT_USER=${GIT_USER:-$USER} -GIT_TEST_URI=git@github.com:${GIT_USER}/clickhouse.github.io.git \ +GIT_PROD_URI=git@github.com:${GIT_USER}/clickhouse.github.io.git \ BASE_DOMAIN=${GIT_USER}-test.clickhouse.com \ - EXTRA_BUILD_ARGS="${@}" \ + EXTRA_BUILD_ARGS="${*}" \ CLOUDFLARE_TOKEN="" \ - HISTORY_SIZE=3 \ - ${BASE_DIR}/release.sh + "${BASE_DIR}/release.sh" diff --git a/docs/tools/release.sh b/docs/tools/release.sh index 3482a0fbcc1..b55841f9da2 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -1,24 +1,24 @@ #!/usr/bin/env bash set -ex -BASE_DIR=$(dirname $(readlink -f $0)) +BASE_DIR=$(dirname "$(readlink -f "$0")") BUILD_DIR="${BASE_DIR}/../build" PUBLISH_DIR="${BASE_DIR}/../publish" BASE_DOMAIN="${BASE_DOMAIN:-content.clickhouse.com}" -GIT_TEST_URI="${GIT_TEST_URI:-git@github.com:ClickHouse/clickhouse-com-content.git}" -GIT_PROD_URI="git@github.com:ClickHouse/clickhouse-website-content.git" +GIT_PROD_URI="${GIT_PROD_URI:-git@github.com:ClickHouse/clickhouse-com-content.git}" EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS:---verbose}" if [[ -z "$1" ]] then source "${BASE_DIR}/venv/bin/activate" + # shellcheck disable=2086 python3 "${BASE_DIR}/build.py" ${EXTRA_BUILD_ARGS} rm -rf "${PUBLISH_DIR}" mkdir "${PUBLISH_DIR}" && cd "${PUBLISH_DIR}" # Will make a repository with website content as the only commit. git init - git remote add origin "${GIT_TEST_URI}" + git remote add origin "${GIT_PROD_URI}" git config user.email "robot-clickhouse@clickhouse.com" git config user.name "robot-clickhouse" @@ -28,7 +28,7 @@ then echo -n "" > README.md echo -n "" > ".nojekyll" cp "${BASE_DIR}/../../LICENSE" . - git add * + git add ./* git add ".nojekyll" git commit --quiet -m "Add new release at $(date)" @@ -40,7 +40,7 @@ then # Turn off logging. set +x - if [[ ! -z "${CLOUDFLARE_TOKEN}" ]] + if [[ -n "${CLOUDFLARE_TOKEN}" ]] then sleep 1m # https://api.cloudflare.com/#zone-purge-files-by-cache-tags,-host-or-prefix diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index b6f2d4549e5..afd6b1a889d 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -2,7 +2,6 @@ Babel==2.9.1 Jinja2==3.0.3 Markdown==3.3.2 MarkupSafe==2.1.1 -MarkupSafe==2.1.1 PyYAML==6.0 Pygments>=2.12.0 beautifulsoup4==4.9.1 @@ -18,7 +17,6 @@ mkdocs-material==8.2.15 mkdocs==1.3.0 mkdocs_material_extensions==1.0.3 packaging==21.3 -pygments==2.12.0 pymdown_extensions==9.4 pyparsing==3.0.9 python-slugify==4.0.1 diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index defc66b0ed9..4a964b81694 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1314,7 +1314,7 @@ int Server::main(const std::vector & /*args*/) global_context->setConfigReloadCallback([&]() { main_config_reloader->reload(); - access_control.reloadUsersConfigs(); + access_control.reload(); }); /// Limit on total number of concurrently executed queries. @@ -1405,6 +1405,7 @@ int Server::main(const std::vector & /*args*/) /// Stop reloading of the main config. This must be done before `global_context->shutdown()` because /// otherwise the reloading may pass a changed config to some destroyed parts of ContextSharedPart. main_config_reloader.reset(); + access_control.stopPeriodicReloading(); async_metrics.stop(); @@ -1628,7 +1629,7 @@ int Server::main(const std::vector & /*args*/) buildLoggers(config(), logger()); main_config_reloader->start(); - access_control.startPeriodicReloadingUsersConfigs(); + access_control.startPeriodicReloading(); if (dns_cache_updater) dns_cache_updater->start(); diff --git a/programs/server/play.html b/programs/server/play.html index 06fc5d8de9a..5d0482c8169 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -81,6 +81,8 @@ { height: 100%; margin: 0; + /* This enables position: sticky on controls */ + overflow: auto; } html @@ -89,9 +91,21 @@ font-family: Liberation Sans, DejaVu Sans, sans-serif, Noto Color Emoji, Apple Color Emoji, Segoe UI Emoji; background: var(--background-color); color: var(--text-color); + } + + body + { + /* This element will show scroll-bar on overflow, and the scroll-bar will be outside of the padding. */ padding: 0.5rem; } + #controls + { + /* When a page will be scrolled horizontally due to large table size, keep controls in place. */ + position: sticky; + left: 0; + } + /* Otherwise Webkit based browsers will display ugly border on focus. */ textarea, input, button { @@ -127,16 +141,12 @@ background-color: var(--element-background-color); } - #query_div - { - /* Make enough space for even huge queries. */ - height: 20%; - } - #query { - height: 100%; - width: 100%; + /* Make enough space for even big queries. */ + height: 20vh; + /* Keeps query text-area's width full screen even when user adjusting the width of the query box. */ + min-width: 100%; } #inputs @@ -380,19 +390,21 @@ -
- -
-
- -
-
- -  (Ctrl/Cmd+Enter) - ⧗ - ✔ - - 🌑🌞 +
+
+ +
+
+ +
+
+ +  (Ctrl/Cmd+Enter) + ⧗ + ✔ + + 🌑🌞 +
diff --git a/src/Access/AccessChangesNotifier.cpp b/src/Access/AccessChangesNotifier.cpp new file mode 100644 index 00000000000..05516285efb --- /dev/null +++ b/src/Access/AccessChangesNotifier.cpp @@ -0,0 +1,122 @@ +#include +#include + + +namespace DB +{ + +AccessChangesNotifier::AccessChangesNotifier() : handlers(std::make_shared()) +{ +} + +AccessChangesNotifier::~AccessChangesNotifier() = default; + +void AccessChangesNotifier::onEntityAdded(const UUID & id, const AccessEntityPtr & new_entity) +{ + std::lock_guard lock{queue_mutex}; + Event event; + event.id = id; + event.entity = new_entity; + event.type = new_entity->getType(); + queue.push(std::move(event)); +} + +void AccessChangesNotifier::onEntityUpdated(const UUID & id, const AccessEntityPtr & changed_entity) +{ + std::lock_guard lock{queue_mutex}; + Event event; + event.id = id; + event.entity = changed_entity; + event.type = changed_entity->getType(); + queue.push(std::move(event)); +} + +void AccessChangesNotifier::onEntityRemoved(const UUID & id, AccessEntityType type) +{ + std::lock_guard lock{queue_mutex}; + Event event; + event.id = id; + event.type = type; + queue.push(std::move(event)); +} + +scope_guard AccessChangesNotifier::subscribeForChanges(AccessEntityType type, const OnChangedHandler & handler) +{ + std::lock_guard lock{handlers->mutex}; + auto & list = handlers->by_type[static_cast(type)]; + list.push_back(handler); + auto handler_it = std::prev(list.end()); + + return [handlers=handlers, type, handler_it] + { + std::lock_guard lock2{handlers->mutex}; + auto & list2 = handlers->by_type[static_cast(type)]; + list2.erase(handler_it); + }; +} + +scope_guard AccessChangesNotifier::subscribeForChanges(const UUID & id, const OnChangedHandler & handler) +{ + std::lock_guard lock{handlers->mutex}; + auto it = handlers->by_id.emplace(id, std::list{}).first; + auto & list = it->second; + list.push_back(handler); + auto handler_it = std::prev(list.end()); + + return [handlers=handlers, it, handler_it] + { + std::lock_guard lock2{handlers->mutex}; + auto & list2 = it->second; + list2.erase(handler_it); + if (list2.empty()) + handlers->by_id.erase(it); + }; +} + + +scope_guard AccessChangesNotifier::subscribeForChanges(const std::vector & ids, const OnChangedHandler & handler) +{ + scope_guard subscriptions; + for (const auto & id : ids) + subscriptions.join(subscribeForChanges(id, handler)); + return subscriptions; +} + +void AccessChangesNotifier::sendNotifications() +{ + /// Only one thread can send notification at any time. + std::lock_guard sending_notifications_lock{sending_notifications}; + + std::unique_lock queue_lock{queue_mutex}; + while (!queue.empty()) + { + auto event = std::move(queue.front()); + queue.pop(); + queue_lock.unlock(); + + std::vector current_handlers; + { + std::lock_guard handlers_lock{handlers->mutex}; + boost::range::copy(handlers->by_type[static_cast(event.type)], std::back_inserter(current_handlers)); + auto it = handlers->by_id.find(event.id); + if (it != handlers->by_id.end()) + boost::range::copy(it->second, std::back_inserter(current_handlers)); + } + + for (const auto & handler : current_handlers) + { + try + { + handler(event.id, event.entity); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + queue_lock.lock(); + } +} + +} diff --git a/src/Access/AccessChangesNotifier.h b/src/Access/AccessChangesNotifier.h new file mode 100644 index 00000000000..46a7cdf26b6 --- /dev/null +++ b/src/Access/AccessChangesNotifier.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include +#include +#include + + +namespace DB +{ + +/// Helper class implementing subscriptions and notifications in access management. +class AccessChangesNotifier +{ +public: + AccessChangesNotifier(); + ~AccessChangesNotifier(); + + using OnChangedHandler + = std::function; + + /// Subscribes for all changes. + /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). + scope_guard subscribeForChanges(AccessEntityType type, const OnChangedHandler & handler); + + template + scope_guard subscribeForChanges(OnChangedHandler handler) + { + return subscribeForChanges(EntityClassT::TYPE, handler); + } + + /// Subscribes for changes of a specific entry. + /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). + scope_guard subscribeForChanges(const UUID & id, const OnChangedHandler & handler); + scope_guard subscribeForChanges(const std::vector & ids, const OnChangedHandler & handler); + + /// Called by access storages after a new access entity has been added. + void onEntityAdded(const UUID & id, const AccessEntityPtr & new_entity); + + /// Called by access storages after an access entity has been changed. + void onEntityUpdated(const UUID & id, const AccessEntityPtr & changed_entity); + + /// Called by access storages after an access entity has been removed. + void onEntityRemoved(const UUID & id, AccessEntityType type); + + /// Sends notifications to subscribers about changes in access entities + /// (added with previous calls onEntityAdded(), onEntityUpdated(), onEntityRemoved()). + void sendNotifications(); + +private: + struct Handlers + { + std::unordered_map> by_id; + std::list by_type[static_cast(AccessEntityType::MAX)]; + std::mutex mutex; + }; + + /// shared_ptr is here for safety because AccessChangesNotifier can be destroyed before all subscriptions are removed. + std::shared_ptr handlers; + + struct Event + { + UUID id; + AccessEntityPtr entity; + AccessEntityType type; + }; + std::queue queue; + std::mutex queue_mutex; + std::mutex sending_notifications; +}; + +} diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index d74695e645e..5cf283ba803 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -14,9 +14,10 @@ #include #include #include +#include #include #include -#include +#include #include #include #include @@ -82,7 +83,7 @@ public: private: const AccessControl & access_control; - Poco::ExpireCache> cache; + Poco::AccessExpireCache> cache; std::mutex mutex; }; @@ -142,7 +143,8 @@ AccessControl::AccessControl() quota_cache(std::make_unique(*this)), settings_profiles_cache(std::make_unique(*this)), external_authenticators(std::make_unique()), - custom_settings_prefixes(std::make_unique()) + custom_settings_prefixes(std::make_unique()), + changes_notifier(std::make_unique()) { } @@ -231,35 +233,6 @@ void AccessControl::addUsersConfigStorage( LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath()); } -void AccessControl::reloadUsersConfigs() -{ - auto storages = getStoragesPtr(); - for (const auto & storage : *storages) - { - if (auto users_config_storage = typeid_cast>(storage)) - users_config_storage->reload(); - } -} - -void AccessControl::startPeriodicReloadingUsersConfigs() -{ - auto storages = getStoragesPtr(); - for (const auto & storage : *storages) - { - if (auto users_config_storage = typeid_cast>(storage)) - users_config_storage->startPeriodicReloading(); - } -} - -void AccessControl::stopPeriodicReloadingUsersConfigs() -{ - auto storages = getStoragesPtr(); - for (const auto & storage : *storages) - { - if (auto users_config_storage = typeid_cast>(storage)) - users_config_storage->stopPeriodicReloading(); - } -} void AccessControl::addReplicatedStorage( const String & storage_name_, @@ -272,10 +245,9 @@ void AccessControl::addReplicatedStorage( if (auto replicated_storage = typeid_cast>(storage)) return; } - auto new_storage = std::make_shared(storage_name_, zookeeper_path_, get_zookeeper_function_); + auto new_storage = std::make_shared(storage_name_, zookeeper_path_, get_zookeeper_function_, *changes_notifier); addStorage(new_storage); LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName()); - new_storage->startup(); } void AccessControl::addDiskStorage(const String & directory_, bool readonly_) @@ -298,7 +270,7 @@ void AccessControl::addDiskStorage(const String & storage_name_, const String & } } } - auto new_storage = std::make_shared(storage_name_, directory_, readonly_); + auto new_storage = std::make_shared(storage_name_, directory_, readonly_, *changes_notifier); addStorage(new_storage); LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath()); } @@ -312,7 +284,7 @@ void AccessControl::addMemoryStorage(const String & storage_name_) if (auto memory_storage = typeid_cast>(storage)) return; } - auto new_storage = std::make_shared(storage_name_); + auto new_storage = std::make_shared(storage_name_, *changes_notifier); addStorage(new_storage); LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName()); } @@ -320,7 +292,7 @@ void AccessControl::addMemoryStorage(const String & storage_name_) void AccessControl::addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) { - auto new_storage = std::make_shared(storage_name_, this, config_, prefix_); + auto new_storage = std::make_shared(storage_name_, *this, config_, prefix_); addStorage(new_storage); LOG_DEBUG(getLogger(), "Added {} access storage '{}', LDAP server name: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getLDAPServerName()); } @@ -423,6 +395,57 @@ void AccessControl::addStoragesFromMainConfig( } +void AccessControl::reload() +{ + MultipleAccessStorage::reload(); + changes_notifier->sendNotifications(); +} + +scope_guard AccessControl::subscribeForChanges(AccessEntityType type, const OnChangedHandler & handler) const +{ + return changes_notifier->subscribeForChanges(type, handler); +} + +scope_guard AccessControl::subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const +{ + return changes_notifier->subscribeForChanges(id, handler); +} + +scope_guard AccessControl::subscribeForChanges(const std::vector & ids, const OnChangedHandler & handler) const +{ + return changes_notifier->subscribeForChanges(ids, handler); +} + +std::optional AccessControl::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) +{ + auto id = MultipleAccessStorage::insertImpl(entity, replace_if_exists, throw_if_exists); + if (id) + changes_notifier->sendNotifications(); + return id; +} + +bool AccessControl::removeImpl(const UUID & id, bool throw_if_not_exists) +{ + bool removed = MultipleAccessStorage::removeImpl(id, throw_if_not_exists); + if (removed) + changes_notifier->sendNotifications(); + return removed; +} + +bool AccessControl::updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) +{ + bool updated = MultipleAccessStorage::updateImpl(id, update_func, throw_if_not_exists); + if (updated) + changes_notifier->sendNotifications(); + return updated; +} + +AccessChangesNotifier & AccessControl::getChangesNotifier() +{ + return *changes_notifier; +} + + UUID AccessControl::authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address) const { try diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index 4ee29aa20c7..cbc71241316 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -3,8 +3,8 @@ #include #include #include +#include #include -#include #include @@ -40,6 +40,7 @@ class SettingsProfilesCache; class SettingsProfileElements; class ClientInfo; class ExternalAuthenticators; +class AccessChangesNotifier; struct Settings; @@ -50,6 +51,7 @@ public: AccessControl(); ~AccessControl() override; + /// Initializes access storage (user directories). void setUpFromMainConfig(const Poco::Util::AbstractConfiguration & config_, const String & config_path_, const zkutil::GetZooKeeper & get_zookeeper_function_); @@ -74,9 +76,6 @@ public: const String & preprocessed_dir_, const zkutil::GetZooKeeper & get_zookeeper_function_ = {}); - void reloadUsersConfigs(); - void startPeriodicReloadingUsersConfigs(); - void stopPeriodicReloadingUsersConfigs(); /// Loads access entities from the directory on the local disk. /// Use that directory to keep created users/roles/etc. void addDiskStorage(const String & directory_, bool readonly_ = false); @@ -106,6 +105,26 @@ public: const String & config_path, const zkutil::GetZooKeeper & get_zookeeper_function); + /// Reloads and updates entities in this storage. This function is used to implement SYSTEM RELOAD CONFIG. + void reload() override; + + using OnChangedHandler = std::function; + + /// Subscribes for all changes. + /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). + scope_guard subscribeForChanges(AccessEntityType type, const OnChangedHandler & handler) const; + + template + scope_guard subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(EntityClassT::TYPE, handler); } + + /// Subscribes for changes of a specific entry. + /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). + scope_guard subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const; + scope_guard subscribeForChanges(const std::vector & ids, const OnChangedHandler & handler) const; + + UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address) const; + void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); + /// Sets the default profile's name. /// The default profile's settings are always applied before any other profile's. void setDefaultProfileName(const String & default_profile_name); @@ -135,9 +154,6 @@ public: void setOnClusterQueriesRequireClusterGrant(bool enable) { on_cluster_queries_require_cluster_grant = enable; } bool doesOnClusterQueriesRequireClusterGrant() const { return on_cluster_queries_require_cluster_grant; } - UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address) const; - void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); - std::shared_ptr getContextAccess( const UUID & user_id, const std::vector & current_roles, @@ -178,10 +194,17 @@ public: const ExternalAuthenticators & getExternalAuthenticators() const; + /// Gets manager of notifications. + AccessChangesNotifier & getChangesNotifier(); + private: class ContextAccessCache; class CustomSettingsPrefixes; + std::optional insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override; + bool removeImpl(const UUID & id, bool throw_if_not_exists) override; + bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; + std::unique_ptr context_access_cache; std::unique_ptr role_cache; std::unique_ptr row_policy_cache; @@ -189,6 +212,7 @@ private: std::unique_ptr settings_profiles_cache; std::unique_ptr external_authenticators; std::unique_ptr custom_settings_prefixes; + std::unique_ptr changes_notifier; std::atomic_bool allow_plaintext_password = true; std::atomic_bool allow_no_password = true; std::atomic_bool users_without_row_policies_can_read_rows = false; diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 28926310c20..46fdba9d65e 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -149,6 +149,21 @@ ContextAccess::ContextAccess(const AccessControl & access_control_, const Params } +ContextAccess::~ContextAccess() +{ + enabled_settings.reset(); + enabled_quota.reset(); + enabled_row_policies.reset(); + access_with_implicit.reset(); + access.reset(); + roles_info.reset(); + subscription_for_roles_changes.reset(); + enabled_roles.reset(); + subscription_for_user_change.reset(); + user.reset(); +} + + void ContextAccess::initialize() { std::lock_guard lock{mutex}; diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index 5742b6a3222..f1c215a4029 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -155,6 +155,8 @@ public: /// without any limitations. This is used for the global context. static std::shared_ptr getFullAccess(); + ~ContextAccess(); + private: friend class AccessControl; ContextAccess() {} /// NOLINT diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 95d58f9da87..57e09d40b35 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -164,13 +165,8 @@ namespace } -DiskAccessStorage::DiskAccessStorage(const String & directory_path_, bool readonly_) - : DiskAccessStorage(STORAGE_TYPE, directory_path_, readonly_) -{ -} - -DiskAccessStorage::DiskAccessStorage(const String & storage_name_, const String & directory_path_, bool readonly_) - : IAccessStorage(storage_name_) +DiskAccessStorage::DiskAccessStorage(const String & storage_name_, const String & directory_path_, bool readonly_, AccessChangesNotifier & changes_notifier_) + : IAccessStorage(storage_name_), changes_notifier(changes_notifier_) { directory_path = makeDirectoryPathCanonical(directory_path_); readonly = readonly_; @@ -199,7 +195,15 @@ DiskAccessStorage::DiskAccessStorage(const String & storage_name_, const String DiskAccessStorage::~DiskAccessStorage() { stopListsWritingThread(); - writeLists(); + + try + { + writeLists(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } } @@ -470,19 +474,16 @@ std::optional DiskAccessStorage::readNameImpl(const UUID & id, bool thro std::optional DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) { - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); - UUID id = generateRandomID(); std::lock_guard lock{mutex}; - if (insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, notifications)) + if (insertNoLock(id, new_entity, replace_if_exists, throw_if_exists)) return id; return std::nullopt; } -bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, Notifications & notifications) +bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) { const String & name = new_entity->getName(); AccessEntityType type = new_entity->getType(); @@ -514,7 +515,7 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne writeAccessEntityToDisk(id, *new_entity); if (name_collision && replace_if_exists) - removeNoLock(it_by_name->second->id, /* throw_if_not_exists = */ false, notifications); + removeNoLock(it_by_name->second->id, /* throw_if_not_exists = */ false); /// Do insertion. auto & entry = entries_by_id[id]; @@ -523,22 +524,20 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne entry.name = name; entry.entity = new_entity; entries_by_name[entry.name] = &entry; - prepareNotifications(id, entry, false, notifications); + + changes_notifier.onEntityAdded(id, new_entity); return true; } bool DiskAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists) { - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); - std::lock_guard lock{mutex}; - return removeNoLock(id, throw_if_not_exists, notifications); + return removeNoLock(id, throw_if_not_exists); } -bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists, Notifications & notifications) +bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists) { auto it = entries_by_id.find(id); if (it == entries_by_id.end()) @@ -559,25 +558,24 @@ bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists, deleteAccessEntityOnDisk(id); /// Do removing. - prepareNotifications(id, entry, true, notifications); + UUID removed_id = id; auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; entries_by_name.erase(entry.name); entries_by_id.erase(it); + + changes_notifier.onEntityRemoved(removed_id, type); return true; } bool DiskAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) { - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); - std::lock_guard lock{mutex}; - return updateNoLock(id, update_func, throw_if_not_exists, notifications); + return updateNoLock(id, update_func, throw_if_not_exists); } -bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, Notifications & notifications) +bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) { auto it = entries_by_id.find(id); if (it == entries_by_id.end()) @@ -626,7 +624,8 @@ bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ entries_by_name[entry.name] = &entry; } - prepareNotifications(id, entry, false, notifications); + changes_notifier.onEntityUpdated(id, new_entity); + return true; } @@ -650,74 +649,4 @@ void DiskAccessStorage::deleteAccessEntityOnDisk(const UUID & id) const throw Exception("Couldn't delete " + file_path, ErrorCodes::FILE_DOESNT_EXIST); } - -void DiskAccessStorage::prepareNotifications(const UUID & id, const Entry & entry, bool remove, Notifications & notifications) const -{ - if (!remove && !entry.entity) - return; - - const AccessEntityPtr entity = remove ? nullptr : entry.entity; - for (const auto & handler : entry.handlers_by_id) - notifications.push_back({handler, id, entity}); - - for (const auto & handler : handlers_by_type[static_cast(entry.type)]) - notifications.push_back({handler, id, entity}); -} - - -scope_guard DiskAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const -{ - std::lock_guard lock{mutex}; - auto it = entries_by_id.find(id); - if (it == entries_by_id.end()) - return {}; - const Entry & entry = it->second; - auto handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler); - - return [this, id, handler_it] - { - std::lock_guard lock2{mutex}; - auto it2 = entries_by_id.find(id); - if (it2 != entries_by_id.end()) - { - const Entry & entry2 = it2->second; - entry2.handlers_by_id.erase(handler_it); - } - }; -} - -scope_guard DiskAccessStorage::subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const -{ - std::lock_guard lock{mutex}; - auto & handlers = handlers_by_type[static_cast(type)]; - handlers.push_back(handler); - auto handler_it = std::prev(handlers.end()); - - return [this, type, handler_it] - { - std::lock_guard lock2{mutex}; - auto & handlers2 = handlers_by_type[static_cast(type)]; - handlers2.erase(handler_it); - }; -} - -bool DiskAccessStorage::hasSubscription(const UUID & id) const -{ - std::lock_guard lock{mutex}; - auto it = entries_by_id.find(id); - if (it != entries_by_id.end()) - { - const Entry & entry = it->second; - return !entry.handlers_by_id.empty(); - } - return false; -} - -bool DiskAccessStorage::hasSubscription(AccessEntityType type) const -{ - std::lock_guard lock{mutex}; - const auto & handlers = handlers_by_type[static_cast(type)]; - return !handlers.empty(); -} - } diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h index 20390dabfa0..7784a80e779 100644 --- a/src/Access/DiskAccessStorage.h +++ b/src/Access/DiskAccessStorage.h @@ -7,14 +7,15 @@ namespace DB { +class AccessChangesNotifier; + /// Loads and saves access entities on a local disk to a specified directory. class DiskAccessStorage : public IAccessStorage { public: static constexpr char STORAGE_TYPE[] = "local directory"; - DiskAccessStorage(const String & storage_name_, const String & directory_path_, bool readonly_ = false); - DiskAccessStorage(const String & directory_path_, bool readonly_ = false); + DiskAccessStorage(const String & storage_name_, const String & directory_path_, bool readonly_, AccessChangesNotifier & changes_notifier_); ~DiskAccessStorage() override; const char * getStorageType() const override { return STORAGE_TYPE; } @@ -27,8 +28,6 @@ public: bool isReadOnly() const override { return readonly; } bool exists(const UUID & id) const override; - bool hasSubscription(const UUID & id) const override; - bool hasSubscription(AccessEntityType type) const override; private: std::optional findImpl(AccessEntityType type, const String & name) const override; @@ -38,8 +37,6 @@ private: std::optional insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override; bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; - scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override; void clear(); bool readLists(); @@ -50,9 +47,9 @@ private: void listsWritingThreadFunc(); void stopListsWritingThread(); - bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, Notifications & notifications); - bool removeNoLock(const UUID & id, bool throw_if_not_exists, Notifications & notifications); - bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, Notifications & notifications); + bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists); + bool removeNoLock(const UUID & id, bool throw_if_not_exists); + bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); AccessEntityPtr readAccessEntityFromDisk(const UUID & id) const; void writeAccessEntityToDisk(const UUID & id, const IAccessEntity & entity) const; @@ -65,11 +62,8 @@ private: String name; AccessEntityType type; mutable AccessEntityPtr entity; /// may be nullptr, if the entity hasn't been loaded yet. - mutable std::list handlers_by_id; }; - void prepareNotifications(const UUID & id, const Entry & entry, bool remove, Notifications & notifications) const; - String directory_path; std::atomic readonly; std::unordered_map entries_by_id; @@ -79,7 +73,7 @@ private: ThreadFromGlobalPool lists_writing_thread; /// List files are written in a separate thread. std::condition_variable lists_writing_thread_should_exit; /// Signals `lists_writing_thread` to exit. bool lists_writing_thread_is_waiting = false; - mutable std::list handlers_by_type[static_cast(AccessEntityType::MAX)]; + AccessChangesNotifier & changes_notifier; mutable std::mutex mutex; }; } diff --git a/src/Access/EnabledRoles.cpp b/src/Access/EnabledRoles.cpp index 282c52a9544..456529da942 100644 --- a/src/Access/EnabledRoles.cpp +++ b/src/Access/EnabledRoles.cpp @@ -6,7 +6,7 @@ namespace DB { -EnabledRoles::EnabledRoles(const Params & params_) : params(params_) +EnabledRoles::EnabledRoles(const Params & params_) : params(params_), handlers(std::make_shared()) { } @@ -15,42 +15,50 @@ EnabledRoles::~EnabledRoles() = default; std::shared_ptr EnabledRoles::getRolesInfo() const { - std::lock_guard lock{mutex}; + std::lock_guard lock{info_mutex}; return info; } scope_guard EnabledRoles::subscribeForChanges(const OnChangeHandler & handler) const { - std::lock_guard lock{mutex}; - handlers.push_back(handler); - auto it = std::prev(handlers.end()); + std::lock_guard lock{handlers->mutex}; + handlers->list.push_back(handler); + auto it = std::prev(handlers->list.end()); - return [this, it] + return [handlers=handlers, it] { - std::lock_guard lock2{mutex}; - handlers.erase(it); + std::lock_guard lock2{handlers->mutex}; + handlers->list.erase(it); }; } -void EnabledRoles::setRolesInfo(const std::shared_ptr & info_, scope_guard & notifications) +void EnabledRoles::setRolesInfo(const std::shared_ptr & info_, scope_guard * notifications) { - std::lock_guard lock{mutex}; - - if (info && info_ && *info == *info_) - return; - - info = info_; - - std::vector handlers_to_notify; - boost::range::copy(handlers, std::back_inserter(handlers_to_notify)); - - notifications.join(scope_guard([info = info, handlers_to_notify = std::move(handlers_to_notify)] { - for (const auto & handler : handlers_to_notify) - handler(info); - })); + std::lock_guard lock{info_mutex}; + if (info && info_ && *info == *info_) + return; + + info = info_; + } + + if (notifications) + { + std::vector handlers_to_notify; + { + std::lock_guard lock{handlers->mutex}; + boost::range::copy(handlers->list, std::back_inserter(handlers_to_notify)); + } + + notifications->join(scope_guard( + [info = info, handlers_to_notify = std::move(handlers_to_notify)] + { + for (const auto & handler : handlers_to_notify) + handler(info); + })); + } } } diff --git a/src/Access/EnabledRoles.h b/src/Access/EnabledRoles.h index 28d1f9ea376..e0d773db343 100644 --- a/src/Access/EnabledRoles.h +++ b/src/Access/EnabledRoles.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -43,12 +44,21 @@ private: friend class RoleCache; explicit EnabledRoles(const Params & params_); - void setRolesInfo(const std::shared_ptr & info_, scope_guard & notifications); + void setRolesInfo(const std::shared_ptr & info_, scope_guard * notifications); const Params params; - mutable std::shared_ptr info; - mutable std::list handlers; - mutable std::mutex mutex; + + std::shared_ptr info; + mutable std::mutex info_mutex; + + struct Handlers + { + std::list list; + std::mutex mutex; + }; + + /// shared_ptr is here for safety because EnabledRoles can be destroyed before all subscriptions are removed. + std::shared_ptr handlers; }; } diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 8c53216c638..6b04355099d 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -410,34 +410,6 @@ bool IAccessStorage::updateImpl(const UUID & id, const UpdateFunc &, bool throw_ } -scope_guard IAccessStorage::subscribeForChanges(AccessEntityType type, const OnChangedHandler & handler) const -{ - return subscribeForChangesImpl(type, handler); -} - - -scope_guard IAccessStorage::subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const -{ - return subscribeForChangesImpl(id, handler); -} - - -scope_guard IAccessStorage::subscribeForChanges(const std::vector & ids, const OnChangedHandler & handler) const -{ - scope_guard subscriptions; - for (const auto & id : ids) - subscriptions.join(subscribeForChangesImpl(id, handler)); - return subscriptions; -} - - -void IAccessStorage::notify(const Notifications & notifications) -{ - for (const auto & [fn, id, new_entity] : notifications) - fn(id, new_entity); -} - - UUID IAccessStorage::authenticate( const Credentials & credentials, const Poco::Net::IPAddress & address, diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 428a0e8f052..5de20cad286 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -22,7 +21,7 @@ enum class AuthenticationType; /// Contains entities, i.e. instances of classes derived from IAccessEntity. /// The implementations of this class MUST be thread-safe. -class IAccessStorage +class IAccessStorage : public boost::noncopyable { public: explicit IAccessStorage(const String & storage_name_) : storage_name(storage_name_) {} @@ -41,6 +40,15 @@ public: /// Returns true if this entity is readonly. virtual bool isReadOnly(const UUID &) const { return isReadOnly(); } + /// Reloads and updates entities in this storage. This function is used to implement SYSTEM RELOAD CONFIG. + virtual void reload() {} + + /// Starts periodic reloading and update of entities in this storage. + virtual void startPeriodicReloading() {} + + /// Stops periodic reloading and update of entities in this storage. + virtual void stopPeriodicReloading() {} + /// Returns the identifiers of all the entities of a specified type contained in the storage. std::vector findAll(AccessEntityType type) const; @@ -130,23 +138,6 @@ public: /// Updates multiple entities in the storage. Returns the list of successfully updated. std::vector tryUpdate(const std::vector & ids, const UpdateFunc & update_func); - using OnChangedHandler = std::function; - - /// Subscribes for all changes. - /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). - scope_guard subscribeForChanges(AccessEntityType type, const OnChangedHandler & handler) const; - - template - scope_guard subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(EntityClassT::TYPE, handler); } - - /// Subscribes for changes of a specific entry. - /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). - scope_guard subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const; - scope_guard subscribeForChanges(const std::vector & ids, const OnChangedHandler & handler) const; - - virtual bool hasSubscription(AccessEntityType type) const = 0; - virtual bool hasSubscription(const UUID & id) const = 0; - /// Finds a user, check the provided credentials and returns the ID of the user if they are valid. /// Throws an exception if no such user or credentials are invalid. UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool allow_no_password, bool allow_plaintext_password) const; @@ -160,8 +151,6 @@ protected: virtual std::optional insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists); virtual bool removeImpl(const UUID & id, bool throw_if_not_exists); virtual bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); - virtual scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const = 0; - virtual scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const = 0; virtual std::optional authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const; virtual bool areCredentialsValid(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const; virtual bool isAddressAllowed(const User & user, const Poco::Net::IPAddress & address) const; @@ -181,9 +170,6 @@ protected: [[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address); [[noreturn]] static void throwInvalidCredentials(); [[noreturn]] static void throwAuthenticationTypeNotAllowed(AuthenticationType auth_type); - using Notification = std::tuple; - using Notifications = std::vector; - static void notify(const Notifications & notifications); private: const String storage_name; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 0fe9e6a1605..480d0050e2a 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -27,10 +27,10 @@ namespace ErrorCodes } -LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControl * access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix) - : IAccessStorage(storage_name_) +LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix) + : IAccessStorage(storage_name_), access_control(access_control_), memory_storage(storage_name_, access_control.getChangesNotifier()) { - setConfiguration(access_control_, config, prefix); + setConfiguration(config, prefix); } @@ -40,7 +40,7 @@ String LDAPAccessStorage::getLDAPServerName() const } -void LDAPAccessStorage::setConfiguration(AccessControl * access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix) +void LDAPAccessStorage::setConfiguration(const Poco::Util::AbstractConfiguration & config, const String & prefix) { std::scoped_lock lock(mutex); @@ -80,7 +80,6 @@ void LDAPAccessStorage::setConfiguration(AccessControl * access_control_, const } } - access_control = access_control_; ldap_server_name = ldap_server_name_cfg; role_search_params.swap(role_search_params_cfg); common_role_names.swap(common_roles_cfg); @@ -91,7 +90,7 @@ void LDAPAccessStorage::setConfiguration(AccessControl * access_control_, const granted_role_names.clear(); granted_role_ids.clear(); - role_change_subscription = access_control->subscribeForChanges( + role_change_subscription = access_control.subscribeForChanges( [this] (const UUID & id, const AccessEntityPtr & entity) { return this->processRoleChange(id, entity); @@ -215,7 +214,7 @@ void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchR auto it = granted_role_ids.find(role_name); if (it == granted_role_ids.end()) { - if (const auto role_id = access_control->find(role_name)) + if (const auto role_id = access_control.find(role_name)) { granted_role_names.insert_or_assign(*role_id, role_name); it = granted_role_ids.insert_or_assign(role_name, *role_id).first; @@ -450,33 +449,6 @@ std::optional LDAPAccessStorage::readNameImpl(const UUID & id, bool thro } -scope_guard LDAPAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const -{ - std::scoped_lock lock(mutex); - return memory_storage.subscribeForChanges(id, handler); -} - - -scope_guard LDAPAccessStorage::subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const -{ - std::scoped_lock lock(mutex); - return memory_storage.subscribeForChanges(type, handler); -} - - -bool LDAPAccessStorage::hasSubscription(const UUID & id) const -{ - std::scoped_lock lock(mutex); - return memory_storage.hasSubscription(id); -} - - -bool LDAPAccessStorage::hasSubscription(AccessEntityType type) const -{ - std::scoped_lock lock(mutex); - return memory_storage.hasSubscription(type); -} - std::optional LDAPAccessStorage::authenticateImpl( const Credentials & credentials, const Poco::Net::IPAddress & address, diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index a86c2fcd35c..df13eff179b 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -32,7 +32,7 @@ class LDAPAccessStorage : public IAccessStorage public: static constexpr char STORAGE_TYPE[] = "ldap"; - explicit LDAPAccessStorage(const String & storage_name_, AccessControl * access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix); + explicit LDAPAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix); virtual ~LDAPAccessStorage() override = default; String getLDAPServerName() const; @@ -42,19 +42,15 @@ public: virtual String getStorageParamsJSON() const override; virtual bool isReadOnly() const override { return true; } virtual bool exists(const UUID & id) const override; - virtual bool hasSubscription(const UUID & id) const override; - virtual bool hasSubscription(AccessEntityType type) const override; private: // IAccessStorage implementations. virtual std::optional findImpl(AccessEntityType type, const String & name) const override; virtual std::vector findAllImpl(AccessEntityType type) const override; virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; virtual std::optional readNameImpl(const UUID & id, bool throw_if_not_exists) const override; - virtual scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - virtual scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override; virtual std::optional authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override; - void setConfiguration(AccessControl * access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix); + void setConfiguration(const Poco::Util::AbstractConfiguration & config, const String & prefix); void processRoleChange(const UUID & id, const AccessEntityPtr & entity); void applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name); @@ -66,7 +62,7 @@ private: // IAccessStorage implementations. const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & role_search_results) const; mutable std::recursive_mutex mutex; - AccessControl * access_control = nullptr; + AccessControl & access_control; String ldap_server_name; LDAPClient::RoleSearchParamsList role_search_params; std::set common_role_names; // role name that should be granted to all users at all times diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 6aa0688ee3e..9ed80f4a64d 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -7,8 +8,8 @@ namespace DB { -MemoryAccessStorage::MemoryAccessStorage(const String & storage_name_) - : IAccessStorage(storage_name_) +MemoryAccessStorage::MemoryAccessStorage(const String & storage_name_, AccessChangesNotifier & changes_notifier_) + : IAccessStorage(storage_name_), changes_notifier(changes_notifier_) { } @@ -63,19 +64,16 @@ AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id, bool throw_if_not std::optional MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) { - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); - UUID id = generateRandomID(); std::lock_guard lock{mutex}; - if (insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, notifications)) + if (insertNoLock(id, new_entity, replace_if_exists, throw_if_exists)) return id; return std::nullopt; } -bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, Notifications & notifications) +bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) { const String & name = new_entity->getName(); AccessEntityType type = new_entity->getType(); @@ -103,7 +101,7 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & if (name_collision && replace_if_exists) { const auto & existing_entry = *(it_by_name->second); - removeNoLock(existing_entry.id, /* throw_if_not_exists = */ false, notifications); + removeNoLock(existing_entry.id, /* throw_if_not_exists = */ false); } /// Do insertion. @@ -111,22 +109,19 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & entry.id = id; entry.entity = new_entity; entries_by_name[name] = &entry; - prepareNotifications(entry, false, notifications); + changes_notifier.onEntityAdded(id, new_entity); return true; } bool MemoryAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists) { - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); - std::lock_guard lock{mutex}; - return removeNoLock(id, throw_if_not_exists, notifications); + return removeNoLock(id, throw_if_not_exists); } -bool MemoryAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists, Notifications & notifications) +bool MemoryAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists) { auto it = entries_by_id.find(id); if (it == entries_by_id.end()) @@ -141,27 +136,25 @@ bool MemoryAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists const String & name = entry.entity->getName(); AccessEntityType type = entry.entity->getType(); - prepareNotifications(entry, true, notifications); - /// Do removing. + UUID removed_id = id; auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; entries_by_name.erase(name); entries_by_id.erase(it); + + changes_notifier.onEntityRemoved(removed_id, type); return true; } bool MemoryAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) { - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); - std::lock_guard lock{mutex}; - return updateNoLock(id, update_func, throw_if_not_exists, notifications); + return updateNoLock(id, update_func, throw_if_not_exists); } -bool MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, Notifications & notifications) +bool MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) { auto it = entries_by_id.find(id); if (it == entries_by_id.end()) @@ -195,7 +188,7 @@ bool MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & updat entries_by_name[new_entity->getName()] = &entry; } - prepareNotifications(entry, false, notifications); + changes_notifier.onEntityUpdated(id, new_entity); return true; } @@ -212,16 +205,8 @@ void MemoryAccessStorage::setAll(const std::vector & all_entiti void MemoryAccessStorage::setAll(const std::vector> & all_entities) { - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); - std::lock_guard lock{mutex}; - setAllNoLock(all_entities, notifications); -} - -void MemoryAccessStorage::setAllNoLock(const std::vector> & all_entities, Notifications & notifications) -{ boost::container::flat_set not_used_ids; std::vector conflicting_ids; @@ -256,7 +241,7 @@ void MemoryAccessStorage::setAllNoLock(const std::vector ids_to_remove = std::move(not_used_ids); boost::range::copy(conflicting_ids, std::inserter(ids_to_remove, ids_to_remove.end())); for (const auto & id : ids_to_remove) - removeNoLock(id, /* throw_if_not_exists = */ false, notifications); + removeNoLock(id, /* throw_if_not_exists = */ false); /// Insert or update entities. for (const auto & [id, entity] : all_entities) @@ -269,84 +254,14 @@ void MemoryAccessStorage::setAllNoLock(const std::vector(entry.entity->getType())]) - notifications.push_back({handler, entry.id, entity}); -} - - -scope_guard MemoryAccessStorage::subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const -{ - std::lock_guard lock{mutex}; - auto & handlers = handlers_by_type[static_cast(type)]; - handlers.push_back(handler); - auto handler_it = std::prev(handlers.end()); - - return [this, type, handler_it] - { - std::lock_guard lock2{mutex}; - auto & handlers2 = handlers_by_type[static_cast(type)]; - handlers2.erase(handler_it); - }; -} - - -scope_guard MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const -{ - std::lock_guard lock{mutex}; - auto it = entries_by_id.find(id); - if (it == entries_by_id.end()) - return {}; - const Entry & entry = it->second; - auto handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler); - - return [this, id, handler_it] - { - std::lock_guard lock2{mutex}; - auto it2 = entries_by_id.find(id); - if (it2 != entries_by_id.end()) - { - const Entry & entry2 = it2->second; - entry2.handlers_by_id.erase(handler_it); - } - }; -} - - -bool MemoryAccessStorage::hasSubscription(const UUID & id) const -{ - std::lock_guard lock{mutex}; - auto it = entries_by_id.find(id); - if (it != entries_by_id.end()) - { - const Entry & entry = it->second; - return !entry.handlers_by_id.empty(); - } - return false; -} - - -bool MemoryAccessStorage::hasSubscription(AccessEntityType type) const -{ - std::lock_guard lock{mutex}; - const auto & handlers = handlers_by_type[static_cast(type)]; - return !handlers.empty(); -} } diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h index f497067bd50..690383c6941 100644 --- a/src/Access/MemoryAccessStorage.h +++ b/src/Access/MemoryAccessStorage.h @@ -9,13 +9,15 @@ namespace DB { +class AccessChangesNotifier; + /// Implementation of IAccessStorage which keeps all data in memory. class MemoryAccessStorage : public IAccessStorage { public: static constexpr char STORAGE_TYPE[] = "memory"; - explicit MemoryAccessStorage(const String & storage_name_ = STORAGE_TYPE); + explicit MemoryAccessStorage(const String & storage_name_, AccessChangesNotifier & changes_notifier_); const char * getStorageType() const override { return STORAGE_TYPE; } @@ -24,8 +26,6 @@ public: void setAll(const std::vector> & all_entities); bool exists(const UUID & id) const override; - bool hasSubscription(const UUID & id) const override; - bool hasSubscription(AccessEntityType type) const override; private: std::optional findImpl(AccessEntityType type, const String & name) const override; @@ -34,25 +34,20 @@ private: std::optional insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override; bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; - scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override; + + bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists); + bool removeNoLock(const UUID & id, bool throw_if_not_exists); + bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); struct Entry { UUID id; AccessEntityPtr entity; - mutable std::list handlers_by_id; }; - bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, Notifications & notifications); - bool removeNoLock(const UUID & id, bool throw_if_not_exists, Notifications & notifications); - bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, Notifications & notifications); - void setAllNoLock(const std::vector> & all_entities, Notifications & notifications); - void prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const; - - mutable std::recursive_mutex mutex; + mutable std::mutex mutex; std::unordered_map entries_by_id; /// We want to search entries both by ID and by the pair of name and type. std::unordered_map entries_by_name_and_type[static_cast(AccessEntityType::MAX)]; - mutable std::list handlers_by_type[static_cast(AccessEntityType::MAX)]; + AccessChangesNotifier & changes_notifier; }; } diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index d71e46c8523..ce4c9f3fd01 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -45,7 +45,6 @@ void MultipleAccessStorage::setStorages(const std::vector & storages std::unique_lock lock{mutex}; nested_storages = std::make_shared(storages); ids_cache.reset(); - updateSubscriptionsToNestedStorages(lock); } void MultipleAccessStorage::addStorage(const StoragePtr & new_storage) @@ -56,7 +55,6 @@ void MultipleAccessStorage::addStorage(const StoragePtr & new_storage) auto new_storages = std::make_shared(*nested_storages); new_storages->push_back(new_storage); nested_storages = new_storages; - updateSubscriptionsToNestedStorages(lock); } void MultipleAccessStorage::removeStorage(const StoragePtr & storage_to_remove) @@ -70,7 +68,6 @@ void MultipleAccessStorage::removeStorage(const StoragePtr & storage_to_remove) new_storages->erase(new_storages->begin() + index); nested_storages = new_storages; ids_cache.reset(); - updateSubscriptionsToNestedStorages(lock); } std::vector MultipleAccessStorage::getStorages() @@ -225,6 +222,28 @@ bool MultipleAccessStorage::isReadOnly(const UUID & id) const } +void MultipleAccessStorage::reload() +{ + auto storages = getStoragesInternal(); + for (const auto & storage : *storages) + storage->reload(); +} + +void MultipleAccessStorage::startPeriodicReloading() +{ + auto storages = getStoragesInternal(); + for (const auto & storage : *storages) + storage->startPeriodicReloading(); +} + +void MultipleAccessStorage::stopPeriodicReloading() +{ + auto storages = getStoragesInternal(); + for (const auto & storage : *storages) + storage->stopPeriodicReloading(); +} + + std::optional MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) { std::shared_ptr storage_for_insertion; @@ -310,145 +329,6 @@ bool MultipleAccessStorage::updateImpl(const UUID & id, const UpdateFunc & updat } -scope_guard MultipleAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const -{ - auto storage = findStorage(id); - if (!storage) - return {}; - return storage->subscribeForChanges(id, handler); -} - - -bool MultipleAccessStorage::hasSubscription(const UUID & id) const -{ - auto storages = getStoragesInternal(); - for (const auto & storage : *storages) - { - if (storage->hasSubscription(id)) - return true; - } - return false; -} - - -scope_guard MultipleAccessStorage::subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const -{ - std::unique_lock lock{mutex}; - auto & handlers = handlers_by_type[static_cast(type)]; - handlers.push_back(handler); - auto handler_it = std::prev(handlers.end()); - if (handlers.size() == 1) - updateSubscriptionsToNestedStorages(lock); - - return [this, type, handler_it] - { - std::unique_lock lock2{mutex}; - auto & handlers2 = handlers_by_type[static_cast(type)]; - handlers2.erase(handler_it); - if (handlers2.empty()) - updateSubscriptionsToNestedStorages(lock2); - }; -} - - -bool MultipleAccessStorage::hasSubscription(AccessEntityType type) const -{ - std::lock_guard lock{mutex}; - const auto & handlers = handlers_by_type[static_cast(type)]; - return !handlers.empty(); -} - - -/// Updates subscriptions to nested storages. -/// We need the subscriptions to the nested storages if someone has subscribed to us. -/// If any of the nested storages is changed we call our subscribers. -void MultipleAccessStorage::updateSubscriptionsToNestedStorages(std::unique_lock & lock) const -{ - /// lock is already locked. - - std::vector> added_subscriptions[static_cast(AccessEntityType::MAX)]; - std::vector removed_subscriptions; - - for (auto type : collections::range(AccessEntityType::MAX)) - { - auto & handlers = handlers_by_type[static_cast(type)]; - auto & subscriptions = subscriptions_to_nested_storages[static_cast(type)]; - if (handlers.empty()) - { - /// None has subscribed to us, we need no subscriptions to the nested storages. - for (auto & subscription : subscriptions | boost::adaptors::map_values) - removed_subscriptions.push_back(std::move(subscription)); - subscriptions.clear(); - } - else - { - /// Someone has subscribed to us, now we need to have a subscription to each nested storage. - for (auto it = subscriptions.begin(); it != subscriptions.end();) - { - const auto & storage = it->first; - auto & subscription = it->second; - if (boost::range::find(*nested_storages, storage) == nested_storages->end()) - { - removed_subscriptions.push_back(std::move(subscription)); - it = subscriptions.erase(it); - } - else - ++it; - } - - for (const auto & storage : *nested_storages) - { - if (!subscriptions.contains(storage)) - added_subscriptions[static_cast(type)].push_back({storage, nullptr}); - } - } - } - - /// Unlock the mutex temporarily because it's much better to subscribe to the nested storages - /// with the mutex unlocked. - lock.unlock(); - removed_subscriptions.clear(); - - for (auto type : collections::range(AccessEntityType::MAX)) - { - if (!added_subscriptions[static_cast(type)].empty()) - { - auto on_changed = [this, type](const UUID & id, const AccessEntityPtr & entity) - { - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); - std::lock_guard lock2{mutex}; - for (const auto & handler : handlers_by_type[static_cast(type)]) - notifications.push_back({handler, id, entity}); - }; - for (auto & [storage, subscription] : added_subscriptions[static_cast(type)]) - subscription = storage->subscribeForChanges(type, on_changed); - } - } - - /// Lock the mutex again to store added subscriptions to the nested storages. - lock.lock(); - - for (auto type : collections::range(AccessEntityType::MAX)) - { - if (!added_subscriptions[static_cast(type)].empty()) - { - auto & subscriptions = subscriptions_to_nested_storages[static_cast(type)]; - for (auto & [storage, subscription] : added_subscriptions[static_cast(type)]) - { - if (!subscriptions.contains(storage) && (boost::range::find(*nested_storages, storage) != nested_storages->end()) - && !handlers_by_type[static_cast(type)].empty()) - { - subscriptions.emplace(std::move(storage), std::move(subscription)); - } - } - } - } - - lock.unlock(); -} - - std::optional MultipleAccessStorage::authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 3a47163af6f..61a975050b6 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -24,6 +24,10 @@ public: bool isReadOnly() const override; bool isReadOnly(const UUID & id) const override; + void reload() override; + void startPeriodicReloading() override; + void stopPeriodicReloading() override; + void setStorages(const std::vector & storages); void addStorage(const StoragePtr & new_storage); void removeStorage(const StoragePtr & storage_to_remove); @@ -37,8 +41,6 @@ public: StoragePtr getStorage(const UUID & id); bool exists(const UUID & id) const override; - bool hasSubscription(const UUID & id) const override; - bool hasSubscription(AccessEntityType type) const override; protected: std::optional findImpl(AccessEntityType type, const String & name) const override; @@ -48,19 +50,14 @@ protected: std::optional insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override; bool removeImpl(const UUID & id, bool throw_if_not_exists) override; bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override; - scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override; std::optional authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override; private: using Storages = std::vector; std::shared_ptr getStoragesInternal() const; - void updateSubscriptionsToNestedStorages(std::unique_lock & lock) const; std::shared_ptr nested_storages; mutable LRUCache ids_cache; - mutable std::list handlers_by_type[static_cast(AccessEntityType::MAX)]; - mutable std::unordered_map subscriptions_to_nested_storages[static_cast(AccessEntityType::MAX)]; mutable std::mutex mutex; }; diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index e56fad720be..d3d1ee3fb6b 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -1,12 +1,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -30,11 +32,13 @@ static UUID parseUUID(const String & text) ReplicatedAccessStorage::ReplicatedAccessStorage( const String & storage_name_, const String & zookeeper_path_, - zkutil::GetZooKeeper get_zookeeper_) + zkutil::GetZooKeeper get_zookeeper_, + AccessChangesNotifier & changes_notifier_) : IAccessStorage(storage_name_) , zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) - , refresh_queue(std::numeric_limits::max()) + , watched_queue(std::make_shared>(std::numeric_limits::max())) + , changes_notifier(changes_notifier_) { if (zookeeper_path.empty()) throw Exception("ZooKeeper path must be non-empty", ErrorCodes::BAD_ARGUMENTS); @@ -45,29 +49,30 @@ ReplicatedAccessStorage::ReplicatedAccessStorage( /// If zookeeper chroot prefix is used, path should start with '/', because chroot concatenates without it. if (zookeeper_path.front() != '/') zookeeper_path = "/" + zookeeper_path; + + initializeZookeeper(); } ReplicatedAccessStorage::~ReplicatedAccessStorage() { - ReplicatedAccessStorage::shutdown(); + stopWatchingThread(); } - -void ReplicatedAccessStorage::startup() +void ReplicatedAccessStorage::startWatchingThread() { - initializeZookeeper(); - worker_thread = ThreadFromGlobalPool(&ReplicatedAccessStorage::runWorkerThread, this); + bool prev_watching_flag = watching.exchange(true); + if (!prev_watching_flag) + watching_thread = ThreadFromGlobalPool(&ReplicatedAccessStorage::runWatchingThread, this); } -void ReplicatedAccessStorage::shutdown() +void ReplicatedAccessStorage::stopWatchingThread() { - bool prev_stop_flag = stop_flag.exchange(true); - if (!prev_stop_flag) + bool prev_watching_flag = watching.exchange(false); + if (prev_watching_flag) { - refresh_queue.finish(); - - if (worker_thread.joinable()) - worker_thread.join(); + watched_queue->finish(); + if (watching_thread.joinable()) + watching_thread.join(); } } @@ -105,10 +110,8 @@ std::optional ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & if (!ok) return std::nullopt; - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); std::lock_guard lock{mutex}; - refreshEntityNoLock(zookeeper, id, notifications); + refreshEntityNoLock(zookeeper, id); return id; } @@ -207,10 +210,8 @@ bool ReplicatedAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exis if (!ok) return false; - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); std::lock_guard lock{mutex}; - removeEntityNoLock(id, notifications); + removeEntityNoLock(id); return true; } @@ -261,10 +262,8 @@ bool ReplicatedAccessStorage::updateImpl(const UUID & id, const UpdateFunc & upd if (!ok) return false; - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); std::lock_guard lock{mutex}; - refreshEntityNoLock(zookeeper, id, notifications); + refreshEntityNoLock(zookeeper, id); return true; } @@ -328,16 +327,18 @@ bool ReplicatedAccessStorage::updateZooKeeper(const zkutil::ZooKeeperPtr & zooke } -void ReplicatedAccessStorage::runWorkerThread() +void ReplicatedAccessStorage::runWatchingThread() { - LOG_DEBUG(getLogger(), "Started worker thread"); - while (!stop_flag) + LOG_DEBUG(getLogger(), "Started watching thread"); + setThreadName("ReplACLWatch"); + while (watching) { try { if (!initialized) initializeZookeeper(); - refresh(); + if (refresh()) + changes_notifier.sendNotifications(); } catch (...) { @@ -353,7 +354,7 @@ void ReplicatedAccessStorage::resetAfterError() initialized = false; UUID id; - while (refresh_queue.tryPop(id)) {} + while (watched_queue->tryPop(id)) {} std::lock_guard lock{mutex}; for (const auto type : collections::range(AccessEntityType::MAX)) @@ -389,21 +390,20 @@ void ReplicatedAccessStorage::createRootNodes(const zkutil::ZooKeeperPtr & zooke } } -void ReplicatedAccessStorage::refresh() +bool ReplicatedAccessStorage::refresh() { UUID id; - if (refresh_queue.tryPop(id, /* timeout_ms: */ 10000)) - { - if (stop_flag) - return; + if (!watched_queue->tryPop(id, /* timeout_ms: */ 10000)) + return false; - auto zookeeper = get_zookeeper(); + auto zookeeper = get_zookeeper(); - if (id == UUIDHelpers::Nil) - refreshEntities(zookeeper); - else - refreshEntity(zookeeper, id); - } + if (id == UUIDHelpers::Nil) + refreshEntities(zookeeper); + else + refreshEntity(zookeeper, id); + + return true; } @@ -412,9 +412,9 @@ void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zooke LOG_DEBUG(getLogger(), "Refreshing entities list"); const String zookeeper_uuids_path = zookeeper_path + "/uuid"; - auto watch_entities_list = [this](const Coordination::WatchResponse &) + auto watch_entities_list = [watched_queue = watched_queue](const Coordination::WatchResponse &) { - [[maybe_unused]] bool push_result = refresh_queue.push(UUIDHelpers::Nil); + [[maybe_unused]] bool push_result = watched_queue->push(UUIDHelpers::Nil); }; Coordination::Stat stat; const auto entity_uuid_strs = zookeeper->getChildrenWatch(zookeeper_uuids_path, &stat, watch_entities_list); @@ -424,8 +424,6 @@ void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zooke for (const String & entity_uuid_str : entity_uuid_strs) entity_uuids.insert(parseUUID(entity_uuid_str)); - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); std::lock_guard lock{mutex}; std::vector entities_to_remove; @@ -437,14 +435,14 @@ void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zooke entities_to_remove.push_back(entity_uuid); } for (const auto & entity_uuid : entities_to_remove) - removeEntityNoLock(entity_uuid, notifications); + removeEntityNoLock(entity_uuid); /// Locally add entities that were added to ZooKeeper for (const auto & entity_uuid : entity_uuids) { const auto it = entries_by_id.find(entity_uuid); if (it == entries_by_id.end()) - refreshEntityNoLock(zookeeper, entity_uuid, notifications); + refreshEntityNoLock(zookeeper, entity_uuid); } LOG_DEBUG(getLogger(), "Refreshing entities list finished"); @@ -452,21 +450,18 @@ void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zooke void ReplicatedAccessStorage::refreshEntity(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) { - Notifications notifications; - SCOPE_EXIT({ notify(notifications); }); std::lock_guard lock{mutex}; - - refreshEntityNoLock(zookeeper, id, notifications); + refreshEntityNoLock(zookeeper, id); } -void ReplicatedAccessStorage::refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, Notifications & notifications) +void ReplicatedAccessStorage::refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) { LOG_DEBUG(getLogger(), "Refreshing entity {}", toString(id)); - const auto watch_entity = [this, id](const Coordination::WatchResponse & response) + const auto watch_entity = [watched_queue = watched_queue, id](const Coordination::WatchResponse & response) { if (response.type == Coordination::Event::CHANGED) - [[maybe_unused]] bool push_result = refresh_queue.push(id); + [[maybe_unused]] bool push_result = watched_queue->push(id); }; Coordination::Stat entity_stat; const String entity_path = zookeeper_path + "/uuid/" + toString(id); @@ -475,16 +470,16 @@ void ReplicatedAccessStorage::refreshEntityNoLock(const zkutil::ZooKeeperPtr & z if (exists) { const AccessEntityPtr entity = deserializeAccessEntity(entity_definition, entity_path); - setEntityNoLock(id, entity, notifications); + setEntityNoLock(id, entity); } else { - removeEntityNoLock(id, notifications); + removeEntityNoLock(id); } } -void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntityPtr & entity, Notifications & notifications) +void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntityPtr & entity) { LOG_DEBUG(getLogger(), "Setting id {} to entity named {}", toString(id), entity->getName()); const AccessEntityType type = entity->getType(); @@ -494,12 +489,14 @@ void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntit auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; if (auto it = entries_by_name.find(name); it != entries_by_name.end() && it->second->id != id) { - removeEntityNoLock(it->second->id, notifications); + removeEntityNoLock(it->second->id); } /// If the entity already exists under a different type+name, remove old type+name + bool existed_before = false; if (auto it = entries_by_id.find(id); it != entries_by_id.end()) { + existed_before = true; const AccessEntityPtr & existing_entity = it->second.entity; const AccessEntityType existing_type = existing_entity->getType(); const String & existing_name = existing_entity->getName(); @@ -514,11 +511,18 @@ void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntit entry.id = id; entry.entity = entity; entries_by_name[name] = &entry; - prepareNotifications(entry, false, notifications); + + if (initialized) + { + if (existed_before) + changes_notifier.onEntityUpdated(id, entity); + else + changes_notifier.onEntityAdded(id, entity); + } } -void ReplicatedAccessStorage::removeEntityNoLock(const UUID & id, Notifications & notifications) +void ReplicatedAccessStorage::removeEntityNoLock(const UUID & id) { LOG_DEBUG(getLogger(), "Removing entity with id {}", toString(id)); const auto it = entries_by_id.find(id); @@ -531,7 +535,6 @@ void ReplicatedAccessStorage::removeEntityNoLock(const UUID & id, Notifications const Entry & entry = it->second; const AccessEntityType type = entry.entity->getType(); const String & name = entry.entity->getName(); - prepareNotifications(entry, true, notifications); auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; const auto name_it = entries_by_name.find(name); @@ -542,8 +545,11 @@ void ReplicatedAccessStorage::removeEntityNoLock(const UUID & id, Notifications else entries_by_name.erase(name); + UUID removed_id = id; entries_by_id.erase(id); LOG_DEBUG(getLogger(), "Removed entity with id {}", toString(id)); + + changes_notifier.onEntityRemoved(removed_id, type); } @@ -594,73 +600,4 @@ AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id, bool throw_if return entry.entity; } - -void ReplicatedAccessStorage::prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const -{ - const AccessEntityPtr entity = remove ? nullptr : entry.entity; - for (const auto & handler : entry.handlers_by_id) - notifications.push_back({handler, entry.id, entity}); - - for (const auto & handler : handlers_by_type[static_cast(entry.entity->getType())]) - notifications.push_back({handler, entry.id, entity}); -} - - -scope_guard ReplicatedAccessStorage::subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const -{ - std::lock_guard lock{mutex}; - auto & handlers = handlers_by_type[static_cast(type)]; - handlers.push_back(handler); - auto handler_it = std::prev(handlers.end()); - - return [this, type, handler_it] - { - std::lock_guard lock2{mutex}; - auto & handlers2 = handlers_by_type[static_cast(type)]; - handlers2.erase(handler_it); - }; -} - - -scope_guard ReplicatedAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const -{ - std::lock_guard lock{mutex}; - const auto it = entries_by_id.find(id); - if (it == entries_by_id.end()) - return {}; - const Entry & entry = it->second; - auto handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler); - - return [this, id, handler_it] - { - std::lock_guard lock2{mutex}; - auto it2 = entries_by_id.find(id); - if (it2 != entries_by_id.end()) - { - const Entry & entry2 = it2->second; - entry2.handlers_by_id.erase(handler_it); - } - }; -} - - -bool ReplicatedAccessStorage::hasSubscription(const UUID & id) const -{ - std::lock_guard lock{mutex}; - const auto & it = entries_by_id.find(id); - if (it != entries_by_id.end()) - { - const Entry & entry = it->second; - return !entry.handlers_by_id.empty(); - } - return false; -} - - -bool ReplicatedAccessStorage::hasSubscription(AccessEntityType type) const -{ - std::lock_guard lock{mutex}; - const auto & handlers = handlers_by_type[static_cast(type)]; - return !handlers.empty(); -} } diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h index 8fdd24b6d54..f9f579e2ba7 100644 --- a/src/Access/ReplicatedAccessStorage.h +++ b/src/Access/ReplicatedAccessStorage.h @@ -18,32 +18,33 @@ namespace DB { +class AccessChangesNotifier; + /// Implementation of IAccessStorage which keeps all data in zookeeper. class ReplicatedAccessStorage : public IAccessStorage { public: static constexpr char STORAGE_TYPE[] = "replicated"; - ReplicatedAccessStorage(const String & storage_name, const String & zookeeper_path, zkutil::GetZooKeeper get_zookeeper); + ReplicatedAccessStorage(const String & storage_name, const String & zookeeper_path, zkutil::GetZooKeeper get_zookeeper, AccessChangesNotifier & changes_notifier_); virtual ~ReplicatedAccessStorage() override; const char * getStorageType() const override { return STORAGE_TYPE; } - virtual void startup(); - virtual void shutdown(); + void startPeriodicReloading() override { startWatchingThread(); } + void stopPeriodicReloading() override { stopWatchingThread(); } bool exists(const UUID & id) const override; - bool hasSubscription(const UUID & id) const override; - bool hasSubscription(AccessEntityType type) const override; private: String zookeeper_path; zkutil::GetZooKeeper get_zookeeper; std::atomic initialized = false; - std::atomic stop_flag = false; - ThreadFromGlobalPool worker_thread; - ConcurrentBoundedQueue refresh_queue; + + std::atomic watching = false; + ThreadFromGlobalPool watching_thread; + std::shared_ptr> watched_queue; std::optional insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override; bool removeImpl(const UUID & id, bool throw_if_not_exists) override; @@ -53,37 +54,36 @@ private: bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists); bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists); - void runWorkerThread(); - void resetAfterError(); void initializeZookeeper(); void createRootNodes(const zkutil::ZooKeeperPtr & zookeeper); - void refresh(); + void startWatchingThread(); + void stopWatchingThread(); + + void runWatchingThread(); + void resetAfterError(); + + bool refresh(); void refreshEntities(const zkutil::ZooKeeperPtr & zookeeper); void refreshEntity(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id); - void refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, Notifications & notifications); + void refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id); - void setEntityNoLock(const UUID & id, const AccessEntityPtr & entity, Notifications & notifications); - void removeEntityNoLock(const UUID & id, Notifications & notifications); + void setEntityNoLock(const UUID & id, const AccessEntityPtr & entity); + void removeEntityNoLock(const UUID & id); struct Entry { UUID id; AccessEntityPtr entity; - mutable std::list handlers_by_id; }; std::optional findImpl(AccessEntityType type, const String & name) const override; std::vector findAllImpl(AccessEntityType type) const override; AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; - void prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const; - scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override; - mutable std::mutex mutex; std::unordered_map entries_by_id; std::unordered_map entries_by_name_and_type[static_cast(AccessEntityType::MAX)]; - mutable std::list handlers_by_type[static_cast(AccessEntityType::MAX)]; + AccessChangesNotifier & changes_notifier; }; } diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index f0e1435e299..308b771243e 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -66,9 +66,6 @@ RoleCache::~RoleCache() = default; std::shared_ptr RoleCache::getEnabledRoles(const std::vector & roles, const std::vector & roles_with_admin_option) { - /// Declared before `lock` to send notifications after the mutex will be unlocked. - scope_guard notifications; - std::lock_guard lock{mutex}; EnabledRoles::Params params; params.current_roles.insert(roles.begin(), roles.end()); @@ -83,13 +80,13 @@ RoleCache::getEnabledRoles(const std::vector & roles, const std::vector(new EnabledRoles(params)); - collectEnabledRoles(*res, notifications); + collectEnabledRoles(*res, nullptr); enabled_roles.emplace(std::move(params), res); return res; } -void RoleCache::collectEnabledRoles(scope_guard & notifications) +void RoleCache::collectEnabledRoles(scope_guard * notifications) { /// `mutex` is already locked. @@ -107,7 +104,7 @@ void RoleCache::collectEnabledRoles(scope_guard & notifications) } -void RoleCache::collectEnabledRoles(EnabledRoles & enabled, scope_guard & notifications) +void RoleCache::collectEnabledRoles(EnabledRoles & enabled, scope_guard * notifications) { /// `mutex` is already locked. @@ -170,7 +167,7 @@ void RoleCache::roleChanged(const UUID & role_id, const RolePtr & changed_role) return; role_from_cache->first = changed_role; cache.update(role_id, role_from_cache); - collectEnabledRoles(notifications); + collectEnabledRoles(¬ifications); } @@ -181,7 +178,7 @@ void RoleCache::roleRemoved(const UUID & role_id) std::lock_guard lock{mutex}; cache.remove(role_id); - collectEnabledRoles(notifications); + collectEnabledRoles(¬ifications); } } diff --git a/src/Access/RoleCache.h b/src/Access/RoleCache.h index e9c731f1342..51c415d4d1d 100644 --- a/src/Access/RoleCache.h +++ b/src/Access/RoleCache.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include @@ -24,14 +24,14 @@ public: const std::vector & current_roles_with_admin_option); private: - void collectEnabledRoles(scope_guard & notifications); - void collectEnabledRoles(EnabledRoles & enabled, scope_guard & notifications); + void collectEnabledRoles(scope_guard * notifications); + void collectEnabledRoles(EnabledRoles & enabled, scope_guard * notifications); RolePtr getRole(const UUID & role_id); void roleChanged(const UUID & role_id, const RolePtr & changed_role); void roleRemoved(const UUID & role_id); const AccessControl & access_control; - Poco::ExpireCache> cache; + Poco::AccessExpireCache> cache; std::map> enabled_roles; mutable std::mutex mutex; }; diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 712e5393ce7..a6c4388fef8 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -14,9 +15,6 @@ #include #include #include -#include -#include -#include #include #include #include @@ -525,8 +523,8 @@ namespace } } -UsersConfigAccessStorage::UsersConfigAccessStorage(const String & storage_name_, const AccessControl & access_control_) - : IAccessStorage(storage_name_), access_control(access_control_) +UsersConfigAccessStorage::UsersConfigAccessStorage(const String & storage_name_, AccessControl & access_control_) + : IAccessStorage(storage_name_), access_control(access_control_), memory_storage(storage_name_, access_control.getChangesNotifier()) { } @@ -605,9 +603,9 @@ void UsersConfigAccessStorage::load( std::make_shared(), [&](Poco::AutoPtr new_config, bool /*initial_loading*/) { - parseFromConfig(*new_config); - Settings::checkNoSettingNamesAtTopLevel(*new_config, users_config_path); + parseFromConfig(*new_config); + access_control.getChangesNotifier().sendNotifications(); }, /* already_loaded = */ false); } @@ -662,27 +660,4 @@ std::optional UsersConfigAccessStorage::readNameImpl(const UUID & id, bo return memory_storage.readName(id, throw_if_not_exists); } - -scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const -{ - return memory_storage.subscribeForChanges(id, handler); -} - - -scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const -{ - return memory_storage.subscribeForChanges(type, handler); -} - - -bool UsersConfigAccessStorage::hasSubscription(const UUID & id) const -{ - return memory_storage.hasSubscription(id); -} - - -bool UsersConfigAccessStorage::hasSubscription(AccessEntityType type) const -{ - return memory_storage.hasSubscription(type); -} } diff --git a/src/Access/UsersConfigAccessStorage.h b/src/Access/UsersConfigAccessStorage.h index e21eb17f462..5c99bf30160 100644 --- a/src/Access/UsersConfigAccessStorage.h +++ b/src/Access/UsersConfigAccessStorage.h @@ -22,7 +22,7 @@ public: static constexpr char STORAGE_TYPE[] = "users.xml"; - UsersConfigAccessStorage(const String & storage_name_, const AccessControl & access_control_); + UsersConfigAccessStorage(const String & storage_name_, AccessControl & access_control_); ~UsersConfigAccessStorage() override; const char * getStorageType() const override { return STORAGE_TYPE; } @@ -37,13 +37,12 @@ public: const String & include_from_path = {}, const String & preprocessed_dir = {}, const zkutil::GetZooKeeper & get_zookeeper_function = {}); - void reload(); - void startPeriodicReloading(); - void stopPeriodicReloading(); + + void reload() override; + void startPeriodicReloading() override; + void stopPeriodicReloading() override; bool exists(const UUID & id) const override; - bool hasSubscription(const UUID & id) const override; - bool hasSubscription(AccessEntityType type) const override; private: void parseFromConfig(const Poco::Util::AbstractConfiguration & config); @@ -51,10 +50,8 @@ private: std::vector findAllImpl(AccessEntityType type) const override; AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override; std::optional readNameImpl(const UUID & id, bool throw_if_not_exists) const override; - scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override; - const AccessControl & access_control; + AccessControl & access_control; MemoryAccessStorage memory_storage; String path; std::unique_ptr config_reloader; diff --git a/src/Access/tests/gtest_replicated_access_storage.cpp b/src/Access/tests/gtest_replicated_access_storage.cpp index f2052e91749..c780e598b64 100644 --- a/src/Access/tests/gtest_replicated_access_storage.cpp +++ b/src/Access/tests/gtest_replicated_access_storage.cpp @@ -1,5 +1,6 @@ #include #include +#include using namespace DB; @@ -12,18 +13,6 @@ namespace ErrorCodes } -TEST(ReplicatedAccessStorage, ShutdownWithoutStartup) -{ - auto get_zk = []() - { - return std::shared_ptr(); - }; - - auto storage = ReplicatedAccessStorage("replicated", "/clickhouse/access", get_zk); - storage.shutdown(); -} - - TEST(ReplicatedAccessStorage, ShutdownWithFailedStartup) { auto get_zk = []() @@ -31,16 +20,16 @@ TEST(ReplicatedAccessStorage, ShutdownWithFailedStartup) return std::shared_ptr(); }; - auto storage = ReplicatedAccessStorage("replicated", "/clickhouse/access", get_zk); + AccessChangesNotifier changes_notifier; + try { - storage.startup(); + auto storage = ReplicatedAccessStorage("replicated", "/clickhouse/access", get_zk, changes_notifier); } catch (Exception & e) { if (e.code() != ErrorCodes::NO_ZOOKEEPER) throw; } - storage.shutdown(); } diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index acff8e7b90f..03aeda1bb9b 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -59,11 +59,11 @@ struct AggregateFunctionSumData } /// Vectorized version - MULTITARGET_FUNCTION_WRAPPER_AVX2_SSE42(addManyImpl, - MULTITARGET_FH( + MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_HEADER( template void NO_SANITIZE_UNDEFINED NO_INLINE - ), /*addManyImpl*/ MULTITARGET_FB((const Value * __restrict ptr, size_t start, size_t end) /// NOLINT + ), addManyImpl, MULTITARGET_FUNCTION_BODY((const Value * __restrict ptr, size_t start, size_t end) /// NOLINT { ptr += start; size_t count = end - start; @@ -122,11 +122,11 @@ struct AggregateFunctionSumData addManyImpl(ptr, start, end); } - MULTITARGET_FUNCTION_WRAPPER_AVX2_SSE42(addManyConditionalInternalImpl, - MULTITARGET_FH( + MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_HEADER( template void NO_SANITIZE_UNDEFINED NO_INLINE - ), /*addManyConditionalInternalImpl*/ MULTITARGET_FB((const Value * __restrict ptr, const UInt8 * __restrict condition_map, size_t start, size_t end) /// NOLINT + ), addManyConditionalInternalImpl, MULTITARGET_FUNCTION_BODY((const Value * __restrict ptr, const UInt8 * __restrict condition_map, size_t start, size_t end) /// NOLINT { ptr += start; size_t count = end - start; diff --git a/src/Columns/IColumnImpl.h b/src/Columns/IColumnImpl.h index 7e7ff3c32d4..e90503cbad2 100644 --- a/src/Columns/IColumnImpl.h +++ b/src/Columns/IColumnImpl.h @@ -81,7 +81,8 @@ void IColumn::compareImpl(const Derived & rhs, size_t rhs_row_num, if constexpr (use_indexes) { num_indexes = row_indexes->size(); - next_index = indexes = row_indexes->data(); + indexes = row_indexes->data(); + next_index = indexes; } compare_results.resize(num_rows); @@ -100,15 +101,9 @@ void IColumn::compareImpl(const Derived & rhs, size_t rhs_row_num, if constexpr (use_indexes) row = indexes[i]; - int res = compareAt(row, rhs_row_num, rhs, nan_direction_hint); - - /// We need to convert int to Int8. Sometimes comparison return values which do not fit in one byte. - if (res < 0) - compare_results[row] = -1; - else if (res > 0) - compare_results[row] = 1; - else - compare_results[row] = 0; + int res = static_cast(this)->compareAt(row, rhs_row_num, rhs, nan_direction_hint); + assert(res == 1 || res == -1 || res == 0); + compare_results[row] = static_cast(res); if constexpr (reversed) compare_results[row] = -compare_results[row]; @@ -124,7 +119,10 @@ void IColumn::compareImpl(const Derived & rhs, size_t rhs_row_num, } if constexpr (use_indexes) - row_indexes->resize(next_index - row_indexes->data()); + { + size_t equal_row_indexes_size = next_index - row_indexes->data(); + row_indexes->resize(equal_row_indexes_size); + } } template diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 203c4636b72..ce457cda1f2 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -628,6 +628,7 @@ M(657, UNSUPPORTED_MEILISEARCH_TYPE) \ M(658, MEILISEARCH_MISSING_SOME_COLUMNS) \ M(659, HDFS_ERROR) \ + M(659, UNKNOWN_STATUS_OF_TRANSACTION) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 21f605ad353..d0f7af2da6b 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -35,6 +35,18 @@ namespace ErrorCodes extern const int CANNOT_MREMAP; } +void abortOnFailedAssertion(const String & description) +{ + LOG_FATAL(&Poco::Logger::root(), "Logical error: '{}'.", description); + + /// This is to suppress -Wmissing-noreturn + volatile bool always_false = false; + if (always_false) + return; + + abort(); +} + /// - Aborts the process if error code is LOGICAL_ERROR. /// - Increments error codes statistics. void handle_error_code([[maybe_unused]] const std::string & msg, int code, bool remote, const Exception::FramePointers & trace) @@ -44,8 +56,7 @@ void handle_error_code([[maybe_unused]] const std::string & msg, int code, bool #ifdef ABORT_ON_LOGICAL_ERROR if (code == ErrorCodes::LOGICAL_ERROR) { - LOG_FATAL(&Poco::Logger::root(), "Logical error: '{}'.", msg); - abort(); + abortOnFailedAssertion(msg); } #endif diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 086b64bf5f9..253dbe6d65c 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -12,16 +12,14 @@ #include -#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(UNDEFINED_BEHAVIOR_SANITIZER) -#define ABORT_ON_LOGICAL_ERROR -#endif - namespace Poco { class Logger; } namespace DB { +void abortOnFailedAssertion(const String & description); + class Exception : public Poco::Exception { public: diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h index 53f3a7d34b1..a3d38f27c07 100644 --- a/src/Common/OptimizedRegularExpression.h +++ b/src/Common/OptimizedRegularExpression.h @@ -107,3 +107,4 @@ private: }; using OptimizedRegularExpression = OptimizedRegularExpressionImpl; +using OptimizedRegularExpressionSingleThreaded = OptimizedRegularExpressionImpl; diff --git a/src/Common/TargetSpecific.h b/src/Common/TargetSpecific.h index 89c0f467fe3..67d9eb4831d 100644 --- a/src/Common/TargetSpecific.h +++ b/src/Common/TargetSpecific.h @@ -233,8 +233,8 @@ DECLARE_AVX512F_SPECIFIC_CODE( * class TestClass * { * public: - * MULTITARGET_FUNCTION_WRAPPER_AVX2_SSE42(testFunctionImpl, - * MULTITARGET_FH(int), /\*testFunction*\/ MULTITARGET_FB((int value) + * MULTITARGET_FUNCTION_AVX2_SSE42( + * MULTITARGET_FUNCTION_HEADER(int), testFunctionImpl, MULTITARGET_FUNCTION_BODY((int value) * { * return value; * }) @@ -259,15 +259,15 @@ DECLARE_AVX512F_SPECIFIC_CODE( */ /// Function header -#define MULTITARGET_FH(...) __VA_ARGS__ +#define MULTITARGET_FUNCTION_HEADER(...) __VA_ARGS__ /// Function body -#define MULTITARGET_FB(...) __VA_ARGS__ +#define MULTITARGET_FUNCTION_BODY(...) __VA_ARGS__ #if ENABLE_MULTITARGET_CODE && defined(__GNUC__) && defined(__x86_64__) /// NOLINTNEXTLINE -#define MULTITARGET_FUNCTION_WRAPPER_AVX2_SSE42(name, FUNCTION_HEADER, FUNCTION_BODY) \ +#define MULTITARGET_FUNCTION_AVX2_SSE42(FUNCTION_HEADER, name, FUNCTION_BODY) \ FUNCTION_HEADER \ \ AVX2_FUNCTION_SPECIFIC_ATTRIBUTE \ @@ -288,7 +288,7 @@ DECLARE_AVX512F_SPECIFIC_CODE( #else /// NOLINTNEXTLINE -#define MULTITARGET_FUNCTION_WRAPPER_AVX2_SSE42(name, FUNCTION_HEADER, FUNCTION_BODY) \ +#define MULTITARGET_FUNCTION_AVX2_SSE42(FUNCTION_HEADER, name, FUNCTION_BODY) \ FUNCTION_HEADER \ \ name \ diff --git a/src/Compression/CompressedWriteBuffer.cpp b/src/Compression/CompressedWriteBuffer.cpp index 93f163dc1af..6c1dbd9e00c 100644 --- a/src/Compression/CompressedWriteBuffer.cpp +++ b/src/Compression/CompressedWriteBuffer.cpp @@ -22,15 +22,22 @@ void CompressedWriteBuffer::nextImpl() if (!offset()) return; - UInt32 compressed_size = 0; size_t decompressed_size = offset(); UInt32 compressed_reserve_size = codec->getCompressedReserveSize(decompressed_size); - if (out.available() > compressed_reserve_size + CHECKSUM_SIZE) + /** During compression we need buffer with capacity >= compressed_reserve_size + CHECKSUM_SIZE. + * + * If output buffer has necessary capacity, we can compress data directly in output buffer. + * Then we can write checksum at the output buffer begin. + * + * If output buffer does not have necessary capacity. Compress data in temporary buffer. + * Then we can write checksum and temporary buffer in output buffer. + */ + if (out.available() >= compressed_reserve_size + CHECKSUM_SIZE) { char * out_checksum_ptr = out.position(); char * out_compressed_ptr = out.position() + CHECKSUM_SIZE; - compressed_size = codec->compress(working_buffer.begin(), decompressed_size, out_compressed_ptr); + UInt32 compressed_size = codec->compress(working_buffer.begin(), decompressed_size, out_compressed_ptr); CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(out_compressed_ptr, compressed_size); memcpy(out_checksum_ptr, reinterpret_cast(&checksum), CHECKSUM_SIZE); @@ -39,7 +46,7 @@ void CompressedWriteBuffer::nextImpl() else { compressed_buffer.resize(compressed_reserve_size); - compressed_size = codec->compress(working_buffer.begin(), decompressed_size, compressed_buffer.data()); + UInt32 compressed_size = codec->compress(working_buffer.begin(), decompressed_size, compressed_buffer.data()); CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(compressed_buffer.data(), compressed_size); out.write(reinterpret_cast(&checksum), CHECKSUM_SIZE); diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 1f089ba2cb7..d74ad173811 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -466,20 +466,23 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ { if (is_recovering) { + const auto finish_recovering = [&] + { + auto new_params = raft_instance->get_current_params(); + new_params.custom_commit_quorum_size_ = 0; + new_params.custom_election_quorum_size_ = 0; + raft_instance->update_params(new_params); + + LOG_INFO(log, "Recovery is done. You can continue using cluster normally."); + is_recovering = false; + }; + switch (type) { case nuraft::cb_func::HeartBeat: { if (raft_instance->isClusterHealthy()) - { - auto new_params = raft_instance->get_current_params(); - new_params.custom_commit_quorum_size_ = 0; - new_params.custom_election_quorum_size_ = 0; - raft_instance->update_params(new_params); - - LOG_INFO(log, "Recovery is done. You can continue using cluster normally."); - is_recovering = false; - } + finish_recovering(); break; } case nuraft::cb_func::NewConfig: @@ -490,8 +493,19 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ // Because we manually set the config to commit // we need to call the reconfigure also uint64_t log_idx = *static_cast(param->ctx); - if (log_idx == state_manager->load_config()->get_log_idx()) - raft_instance->forceReconfigure(state_manager->load_config()); + + auto config = state_manager->load_config(); + if (log_idx == config->get_log_idx()) + { + raft_instance->forceReconfigure(config); + + // Single node cluster doesn't need to wait for any other nodes + // so we can finish recovering immediately after applying + // new configuration + if (config->get_servers().size() == 1) + finish_recovering(); + } + break; } case nuraft::cb_func::ProcessReq: diff --git a/src/Core/DecimalFunctions.h b/src/Core/DecimalFunctions.h index f08527ee4d5..331df9aa637 100644 --- a/src/Core/DecimalFunctions.h +++ b/src/Core/DecimalFunctions.h @@ -156,7 +156,7 @@ inline DecimalComponents splitWithScaleMultiplier( using T = typename DecimalType::NativeType; const auto whole = decimal.value / scale_multiplier; auto fractional = decimal.value % scale_multiplier; - if (fractional < T(0)) + if (whole && fractional < T(0)) fractional *= T(-1); return {whole, fractional}; @@ -199,7 +199,7 @@ inline typename DecimalType::NativeType getFractionalPartWithScaleMultiplier( /// Anycase we make modulo before compare to make scale_multiplier > 1 unaffected. T result = decimal.value % scale_multiplier; if constexpr (!keep_sign) - if (result < T(0)) + if (decimal.value / scale_multiplier && result < T(0)) result = -result; return result; diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ca79b3a781d..75a30a2d792 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -593,6 +593,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \ M(Bool, count_distinct_optimization, false, "Rewrite count distinct to subquery of group by", 0) \ M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \ + M(TransactionsWaitCSNMode, wait_changes_become_visible_after_commit_mode, TransactionsWaitCSNMode::WAIT_UNKNOWN, "Wait for committed changes to become actually visible in the latest snapshot", 0) \ M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \ M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ // End of COMMON_SETTINGS @@ -638,7 +639,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, output_format_csv_crlf_end_of_line, false, "If it is set true, end of line in CSV format will be \\r\\n instead of \\n.", 0) \ M(Bool, input_format_csv_enum_as_number, false, "Treat inserted enum values in CSV formats as enum indices \\N", 0) \ M(Bool, input_format_csv_arrays_as_nested_csv, false, R"(When reading Array from CSV, expect that its elements were serialized in nested CSV and then put into string. Example: "[""Hello"", ""world"", ""42"""" TV""]". Braces around array can be omitted.)", 0) \ - M(Bool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow, -WithNames, -WithNamesAndTypes and TSKV formats).", 0) \ + M(Bool, input_format_skip_unknown_fields, true, "Skip columns with unknown names from input data (it works for JSONEachRow, -WithNames, -WithNamesAndTypes and TSKV formats).", 0) \ M(Bool, input_format_with_names_use_header, true, "For -WithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.", 0) \ M(Bool, input_format_with_types_use_header, true, "For -WithNamesAndTypes input formats this controls whether format parser should check if data types from the input match data types from the header.", 0) \ M(Bool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).", 0) \ diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index a37c1e9be86..bff1971bad9 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -131,6 +131,11 @@ IMPLEMENT_SETTING_ENUM(ShortCircuitFunctionEvaluation, ErrorCodes::BAD_ARGUMENTS {"force_enable", ShortCircuitFunctionEvaluation::FORCE_ENABLE}, {"disable", ShortCircuitFunctionEvaluation::DISABLE}}) +IMPLEMENT_SETTING_ENUM(TransactionsWaitCSNMode, ErrorCodes::BAD_ARGUMENTS, + {{"async", TransactionsWaitCSNMode::ASYNC}, + {"wait", TransactionsWaitCSNMode::WAIT}, + {"wait_unknown", TransactionsWaitCSNMode::WAIT_UNKNOWN}}) + IMPLEMENT_SETTING_ENUM(EnumComparingMode, ErrorCodes::BAD_ARGUMENTS, {{"by_names", FormatSettings::EnumComparingMode::BY_NAMES}, {"by_values", FormatSettings::EnumComparingMode::BY_VALUES}, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 08091da6d6c..83a65f2a320 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -183,6 +183,15 @@ enum class ShortCircuitFunctionEvaluation DECLARE_SETTING_ENUM(ShortCircuitFunctionEvaluation) +enum class TransactionsWaitCSNMode +{ + ASYNC, + WAIT, + WAIT_UNKNOWN, +}; + +DECLARE_SETTING_ENUM(TransactionsWaitCSNMode) + DECLARE_SETTING_ENUM_WITH_RENAME(EnumComparingMode, FormatSettings::EnumComparingMode) DECLARE_SETTING_ENUM_WITH_RENAME(EscapingRule, FormatSettings::EscapingRule) diff --git a/src/Core/tests/gtest_DecimalFunctions.cpp b/src/Core/tests/gtest_DecimalFunctions.cpp index 7517edda937..1712785488e 100644 --- a/src/Core/tests/gtest_DecimalFunctions.cpp +++ b/src/Core/tests/gtest_DecimalFunctions.cpp @@ -176,7 +176,7 @@ INSTANTIATE_TEST_SUITE_P(Basic, } }, { - "When scale is not 0 and whole part is 0.", + "For positive Decimal value, with scale not 0, and whole part is 0.", 123, 3, { @@ -184,6 +184,16 @@ INSTANTIATE_TEST_SUITE_P(Basic, 123 } }, + { + "For negative Decimal value, with scale not 0, and whole part is 0.", + -123, + 3, + { + 0, + -123 + } + }, + { "For negative Decimal value whole part is negative, fractional is non-negative.", -1234567'89, @@ -216,6 +226,24 @@ INSTANTIATE_TEST_SUITE_P(Basic, 187618332, 123 } + }, + { + "Negative timestamp 1969-12-31 23:59:59.123 UTC", + DateTime64(-877), + 3, + { + 0, + -877 + } + }, + { + "Positive timestamp 1970-01-01 00:00:00.123 UTC", + DateTime64(123), + 3, + { + 0, + 123 + } } }) ); diff --git a/src/DataTypes/Serializations/SerializationNothing.h b/src/DataTypes/Serializations/SerializationNothing.h index 2de93a29763..e46a1e6ed30 100644 --- a/src/DataTypes/Serializations/SerializationNothing.h +++ b/src/DataTypes/Serializations/SerializationNothing.h @@ -16,7 +16,7 @@ class SerializationNothing : public SimpleTextSerialization private: [[noreturn]] static void throwNoSerialization() { - throw Exception("Serialization is not implemented", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Serialization is not implemented for type Nothing", ErrorCodes::NOT_IMPLEMENTED); } public: void serializeBinary(const Field &, WriteBuffer &) const override { throwNoSerialization(); } diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index 0abfc4a618a..883e58f9c8c 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -83,6 +83,9 @@ public: UInt32 getRefCount(const String & path) const override { return delegate->getRefCount(path); } + void syncRevision(UInt64 revision) override { delegate->syncRevision(revision); } + UInt64 getRevision() const override { return delegate->getRevision(); } + protected: Executor & getExecutor() override; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index ce6cc84c3f3..f243f477f2c 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -337,6 +337,14 @@ public: /// other alive harlinks will not be removed. virtual UInt32 getRefCount(const String &) const { return 0; } + /// Revision is an incremental counter of disk operation. + /// Revision currently exisis only in DiskS3. + /// It is used to save current state during backup and restore that state from backup. + /// This method sets current disk revision if it lower than required. + virtual void syncRevision(UInt64) {} + /// Return current disk revision. + virtual UInt64 getRevision() const { return 0; } + protected: friend class DiskDecorator; diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index d54ef02ec7c..145bb4a3d66 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -1,6 +1,25 @@ #include +#if USE_AWS_S3 + #include +#include +#include +#include +#include +#include "Disks/DiskFactory.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace DB { @@ -130,3 +149,4 @@ std::unique_ptr getClient(const Poco::Util::AbstractConfigura } } +>>>>>> master:src/Disks/S3/registerDiskS3.cpp diff --git a/src/Formats/EscapingRuleUtils.cpp b/src/Formats/EscapingRuleUtils.cpp index e4b655cdcf9..5aab8909a0c 100644 --- a/src/Formats/EscapingRuleUtils.cpp +++ b/src/Formats/EscapingRuleUtils.cpp @@ -71,7 +71,7 @@ String escapingRuleToString(FormatSettings::EscapingRule escaping_rule) void skipFieldByEscapingRule(ReadBuffer & buf, FormatSettings::EscapingRule escaping_rule, const FormatSettings & format_settings) { - String tmp; + NullOutput out; constexpr const char * field_name = ""; constexpr size_t field_name_len = 16; switch (escaping_rule) @@ -80,19 +80,19 @@ void skipFieldByEscapingRule(ReadBuffer & buf, FormatSettings::EscapingRule esca /// Empty field, just skip spaces break; case FormatSettings::EscapingRule::Escaped: - readEscapedString(tmp, buf); + readEscapedStringInto(out, buf); break; case FormatSettings::EscapingRule::Quoted: - readQuotedField(tmp, buf); + readQuotedFieldInto(out, buf); break; case FormatSettings::EscapingRule::CSV: - readCSVString(tmp, buf, format_settings.csv); + readCSVStringInto(out, buf, format_settings.csv); break; case FormatSettings::EscapingRule::JSON: skipJSONField(buf, StringRef(field_name, field_name_len)); break; case FormatSettings::EscapingRule::Raw: - readString(tmp, buf); + readStringInto(out, buf); break; default: __builtin_unreachable(); diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 4c1b23a75ab..644e4d3ecfd 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -541,19 +541,19 @@ void FormatFactory::markOutputFormatSupportsParallelFormatting(const String & na } -void FormatFactory::markFormatAsColumnOriented(const String & name) +void FormatFactory::markFormatSupportsSubsetOfColumns(const String & name) { - auto & target = dict[name].is_column_oriented; + auto & target = dict[name].supports_subset_of_columns; if (target) - throw Exception("FormatFactory: Format " + name + " is already marked as column oriented", ErrorCodes::LOGICAL_ERROR); + throw Exception("FormatFactory: Format " + name + " is already marked as supporting subset of columns", ErrorCodes::LOGICAL_ERROR); target = true; } -bool FormatFactory::checkIfFormatIsColumnOriented(const String & name) +bool FormatFactory::checkIfFormatSupportsSubsetOfColumns(const String & name) const { const auto & target = getCreators(name); - return target.is_column_oriented; + return target.supports_subset_of_columns; } bool FormatFactory::isInputFormat(const String & name) const @@ -568,19 +568,19 @@ bool FormatFactory::isOutputFormat(const String & name) const return it != dict.end() && it->second.output_creator; } -bool FormatFactory::checkIfFormatHasSchemaReader(const String & name) +bool FormatFactory::checkIfFormatHasSchemaReader(const String & name) const { const auto & target = getCreators(name); return bool(target.schema_reader_creator); } -bool FormatFactory::checkIfFormatHasExternalSchemaReader(const String & name) +bool FormatFactory::checkIfFormatHasExternalSchemaReader(const String & name) const { const auto & target = getCreators(name); return bool(target.external_schema_reader_creator); } -bool FormatFactory::checkIfFormatHasAnySchemaReader(const String & name) +bool FormatFactory::checkIfFormatHasAnySchemaReader(const String & name) const { return checkIfFormatHasSchemaReader(name) || checkIfFormatHasExternalSchemaReader(name); } diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index f7d3c23d3b4..8e949a3e367 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -108,7 +108,7 @@ private: SchemaReaderCreator schema_reader_creator; ExternalSchemaReaderCreator external_schema_reader_creator; bool supports_parallel_formatting{false}; - bool is_column_oriented{false}; + bool supports_subset_of_columns{false}; NonTrivialPrefixAndSuffixChecker non_trivial_prefix_and_suffix_checker; AppendSupportChecker append_support_checker; }; @@ -194,13 +194,13 @@ public: void registerExternalSchemaReader(const String & name, ExternalSchemaReaderCreator external_schema_reader_creator); void markOutputFormatSupportsParallelFormatting(const String & name); - void markFormatAsColumnOriented(const String & name); + void markFormatSupportsSubsetOfColumns(const String & name); - bool checkIfFormatIsColumnOriented(const String & name); + bool checkIfFormatSupportsSubsetOfColumns(const String & name) const; - bool checkIfFormatHasSchemaReader(const String & name); - bool checkIfFormatHasExternalSchemaReader(const String & name); - bool checkIfFormatHasAnySchemaReader(const String & name); + bool checkIfFormatHasSchemaReader(const String & name) const; + bool checkIfFormatHasExternalSchemaReader(const String & name) const; + bool checkIfFormatHasAnySchemaReader(const String & name) const; const FormatsDictionary & getAllFormats() const { diff --git a/src/Formats/NativeReader.cpp b/src/Formats/NativeReader.cpp index ed3aca43d52..3ad0ce5cfc4 100644 --- a/src/Formats/NativeReader.cpp +++ b/src/Formats/NativeReader.cpp @@ -23,6 +23,7 @@ namespace ErrorCodes extern const int INCORRECT_INDEX; extern const int LOGICAL_ERROR; extern const int CANNOT_READ_ALL_DATA; + extern const int INCORRECT_DATA; } @@ -31,8 +32,8 @@ NativeReader::NativeReader(ReadBuffer & istr_, UInt64 server_revision_) { } -NativeReader::NativeReader(ReadBuffer & istr_, const Block & header_, UInt64 server_revision_) - : istr(istr_), header(header_), server_revision(server_revision_) +NativeReader::NativeReader(ReadBuffer & istr_, const Block & header_, UInt64 server_revision_, bool skip_unknown_columns_) + : istr(istr_), header(header_), server_revision(server_revision_), skip_unknown_columns(skip_unknown_columns_) { } @@ -186,18 +187,29 @@ Block NativeReader::read() column.column = std::move(read_column); + bool use_in_result = true; if (header) { - /// Support insert from old clients without low cardinality type. - auto & header_column = header.getByName(column.name); - if (!header_column.type->equals(*column.type)) + if (header.has(column.name)) { - column.column = recursiveTypeConversion(column.column, column.type, header.safeGetByPosition(i).type); - column.type = header.safeGetByPosition(i).type; + /// Support insert from old clients without low cardinality type. + auto & header_column = header.getByName(column.name); + if (!header_column.type->equals(*column.type)) + { + column.column = recursiveTypeConversion(column.column, column.type, header.safeGetByPosition(i).type); + column.type = header.safeGetByPosition(i).type; + } + } + else + { + if (!skip_unknown_columns) + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown column with name {} found while reading data in Native format", column.name); + use_in_result = false; } } - res.insert(std::move(column)); + if (use_in_result) + res.insert(std::move(column)); if (use_index) ++index_column_it; diff --git a/src/Formats/NativeReader.h b/src/Formats/NativeReader.h index 1f9eb8b9764..3ae53d45faf 100644 --- a/src/Formats/NativeReader.h +++ b/src/Formats/NativeReader.h @@ -24,7 +24,7 @@ public: /// For cases when data structure (header) is known in advance. /// NOTE We may use header for data validation and/or type conversions. It is not implemented. - NativeReader(ReadBuffer & istr_, const Block & header_, UInt64 server_revision_); + NativeReader(ReadBuffer & istr_, const Block & header_, UInt64 server_revision_, bool skip_unknown_columns_ = false); /// For cases when we have an index. It allows to skip columns. Only columns specified in the index will be read. NativeReader(ReadBuffer & istr_, UInt64 server_revision_, @@ -43,6 +43,7 @@ private: ReadBuffer & istr; Block header; UInt64 server_revision; + bool skip_unknown_columns; bool use_index = false; IndexForNativeFormat::Blocks::const_iterator index_block_it; diff --git a/src/Formats/registerWithNamesAndTypes.cpp b/src/Formats/registerWithNamesAndTypes.cpp index cba578b08c7..2dee107844d 100644 --- a/src/Formats/registerWithNamesAndTypes.cpp +++ b/src/Formats/registerWithNamesAndTypes.cpp @@ -10,4 +10,10 @@ void registerWithNamesAndTypes(const std::string & base_format_name, RegisterWit register_func(base_format_name + "WithNamesAndTypes", true, true); } +void markFormatWithNamesAndTypesSupportsSamplingColumns(const std::string & base_format_name, FormatFactory & factory) +{ + factory.markFormatSupportsSubsetOfColumns(base_format_name + "WithNames"); + factory.markFormatSupportsSubsetOfColumns(base_format_name + "WithNamesAndTypes"); +} + } diff --git a/src/Formats/registerWithNamesAndTypes.h b/src/Formats/registerWithNamesAndTypes.h index d8e74e3421e..50a0eee9616 100644 --- a/src/Formats/registerWithNamesAndTypes.h +++ b/src/Formats/registerWithNamesAndTypes.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB { @@ -9,4 +10,6 @@ namespace DB using RegisterWithNamesAndTypesFunc = std::function; void registerWithNamesAndTypes(const std::string & base_format_name, RegisterWithNamesAndTypesFunc register_func); +void markFormatWithNamesAndTypesSupportsSamplingColumns(const std::string & base_format_name, FormatFactory & factory); + } diff --git a/src/Functions/CastOverloadResolver.h b/src/Functions/CastOverloadResolver.h index cff17d810fe..9b579c34923 100644 --- a/src/Functions/CastOverloadResolver.h +++ b/src/Functions/CastOverloadResolver.h @@ -98,6 +98,7 @@ protected: } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForNothing() const override { return false; } bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } private: diff --git a/src/Functions/CountSubstringsImpl.h b/src/Functions/CountSubstringsImpl.h index fc6e4a0e671..c8cef81333a 100644 --- a/src/Functions/CountSubstringsImpl.h +++ b/src/Functions/CountSubstringsImpl.h @@ -26,19 +26,21 @@ struct CountSubstringsImpl static constexpr bool supports_start_pos = true; static constexpr auto name = Name::name; + static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {};} + using ResultType = UInt64; /// Count occurrences of one substring in many strings. static void vectorConstant( - const ColumnString::Chars & data, - const ColumnString::Offsets & offsets, + const ColumnString::Chars & haystack_data, + const ColumnString::Offsets & haystack_offsets, const std::string & needle, const ColumnPtr & start_pos, PaddedPODArray & res) { - const UInt8 * begin = data.data(); + const UInt8 * const begin = haystack_data.data(); + const UInt8 * const end = haystack_data.data() + haystack_data.size(); const UInt8 * pos = begin; - const UInt8 * end = pos + data.size(); /// FIXME: suboptimal memset(&res[0], 0, res.size() * sizeof(res[0])); @@ -52,15 +54,15 @@ struct CountSubstringsImpl while (pos < end && end != (pos = searcher.search(pos, end - pos))) { /// Determine which index it refers to. - while (begin + offsets[i] <= pos) + while (begin + haystack_offsets[i] <= pos) ++i; auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0; /// We check that the entry does not pass through the boundaries of strings. - if (pos + needle.size() < begin + offsets[i]) + if (pos + needle.size() < begin + haystack_offsets[i]) { - auto res_pos = needle.size() + Impl::countChars(reinterpret_cast(begin + offsets[i - 1]), reinterpret_cast(pos)); + auto res_pos = needle.size() + Impl::countChars(reinterpret_cast(begin + haystack_offsets[i - 1]), reinterpret_cast(pos)); if (res_pos >= start) { ++res[i]; @@ -69,14 +71,14 @@ struct CountSubstringsImpl pos += needle.size(); continue; } - pos = begin + offsets[i]; + pos = begin + haystack_offsets[i]; ++i; } } /// Count number of occurrences of substring in string. static void constantConstantScalar( - std::string data, + std::string haystack, std::string needle, UInt64 start_pos, UInt64 & res) @@ -87,9 +89,9 @@ struct CountSubstringsImpl return; auto start = std::max(start_pos, UInt64(1)); - size_t start_byte = Impl::advancePos(data.data(), data.data() + data.size(), start - 1) - data.data(); + size_t start_byte = Impl::advancePos(haystack.data(), haystack.data() + haystack.size(), start - 1) - haystack.data(); size_t new_start_byte; - while ((new_start_byte = data.find(needle, start_byte)) != std::string::npos) + while ((new_start_byte = haystack.find(needle, start_byte)) != std::string::npos) { ++res; /// Intersecting substrings in haystack accounted only once @@ -99,21 +101,21 @@ struct CountSubstringsImpl /// Count number of occurrences of substring in string starting from different positions. static void constantConstant( - std::string data, + std::string haystack, std::string needle, const ColumnPtr & start_pos, PaddedPODArray & res) { - Impl::toLowerIfNeed(data); + Impl::toLowerIfNeed(haystack); Impl::toLowerIfNeed(needle); if (start_pos == nullptr) { - constantConstantScalar(data, needle, 0, res[0]); + constantConstantScalar(haystack, needle, 0, res[0]); return; } - size_t haystack_size = Impl::countChars(data.data(), data.data() + data.size()); + size_t haystack_size = Impl::countChars(haystack.data(), haystack.data() + haystack.size()); size_t size = start_pos != nullptr ? start_pos->size() : 0; for (size_t i = 0; i < size; ++i) @@ -125,7 +127,7 @@ struct CountSubstringsImpl res[i] = 0; continue; } - constantConstantScalar(data, needle, start, res[i]); + constantConstantScalar(haystack, needle, start, res[i]); } } @@ -228,6 +230,12 @@ struct CountSubstringsImpl { throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name); } + + template + static void vectorFixedVector(Args &&...) + { + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name); + } }; } diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index 4dc769b8177..445eb45fd9d 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -42,9 +42,8 @@ struct UnaryOperationImpl using ArrayA = typename ColVecA::Container; using ArrayC = typename ColVecC::Container; - MULTITARGET_FUNCTION_WRAPPER_AVX2_SSE42(vectorImpl, - MULTITARGET_FH( - static void NO_INLINE), /*vectorImpl*/ MULTITARGET_FB((const ArrayA & a, ArrayC & c) /// NOLINT + MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_HEADER(static void NO_INLINE), vectorImpl, MULTITARGET_FUNCTION_BODY((const ArrayA & a, ArrayC & c) /// NOLINT { size_t size = a.size(); for (size_t i = 0; i < size; ++i) @@ -79,9 +78,9 @@ struct UnaryOperationImpl template struct FixedStringUnaryOperationImpl { - MULTITARGET_FUNCTION_WRAPPER_AVX2_SSE42(vectorImpl, - MULTITARGET_FH( - static void NO_INLINE), /*vectorImpl*/ MULTITARGET_FB((const ColumnFixedString::Chars & a, ColumnFixedString::Chars & c) /// NOLINT + MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_HEADER(static void NO_INLINE), vectorImpl, MULTITARGET_FUNCTION_BODY((const ColumnFixedString::Chars & a, /// NOLINT + ColumnFixedString::Chars & c) { size_t size = a.size(); for (size_t i = 0; i < size; ++i) diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 16575e551a7..7bbb1c1096c 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -85,8 +85,9 @@ struct NumComparisonImpl using ContainerA = PaddedPODArray; using ContainerB = PaddedPODArray; - MULTITARGET_FUNCTION_WRAPPER_AVX2_SSE42(vectorVectorImpl, - MULTITARGET_FH(static void), /*vectorVectorImpl*/ MULTITARGET_FB((const ContainerA & a, const ContainerB & b, PaddedPODArray & c) /// NOLINT + MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_HEADER(static void), vectorVectorImpl, MULTITARGET_FUNCTION_BODY(( /// NOLINT + const ContainerA & a, const ContainerB & b, PaddedPODArray & c) { /** GCC 4.8.2 vectorizes a loop only if it is written in this form. * In this case, if you loop through the array index (the code will look simpler), @@ -127,8 +128,9 @@ struct NumComparisonImpl } - MULTITARGET_FUNCTION_WRAPPER_AVX2_SSE42(vectorConstantImpl, - MULTITARGET_FH(static void), /*vectorConstantImpl*/ MULTITARGET_FB((const ContainerA & a, B b, PaddedPODArray & c) /// NOLINT + MULTITARGET_FUNCTION_AVX2_SSE42( + MULTITARGET_FUNCTION_HEADER(static void), vectorConstantImpl, MULTITARGET_FUNCTION_BODY(( /// NOLINT + const ContainerA & a, B b, PaddedPODArray & c) { size_t size = a.size(); const A * __restrict a_pos = a.data(); diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index ba3c9d68241..bffc15cdc57 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -2516,6 +2516,8 @@ protected: } bool useDefaultImplementationForNulls() const override { return false; } + /// CAST(Nothing, T) -> T + bool useDefaultImplementationForNothing() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } diff --git a/src/Functions/FunctionsMiscellaneous.h b/src/Functions/FunctionsMiscellaneous.h index 7e8cab842c8..49da8f49c82 100644 --- a/src/Functions/FunctionsMiscellaneous.h +++ b/src/Functions/FunctionsMiscellaneous.h @@ -50,7 +50,11 @@ public: return expr_columns.getByName(signature->return_name).column; } -bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForNulls() const override { return false; } + /// It's possible if expression_actions contains function that don't use + /// default implementation for Nothing. + /// Example: arrayMap(x -> CAST(x, 'UInt8'), []); + bool useDefaultImplementationForNothing() const override { return false; } private: ExpressionActionsPtr expression_actions; @@ -118,6 +122,10 @@ public: String getName() const override { return "FunctionCapture"; } bool useDefaultImplementationForNulls() const override { return false; } + /// It's possible if expression_actions contains function that don't use + /// default implementation for Nothing and one of captured columns can be Nothing + /// Example: SELECT arrayMap(x -> [x, arrayElement(y, 0)], []), [] as y + bool useDefaultImplementationForNothing() const override { return false; } bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override @@ -247,6 +255,8 @@ public: String getName() const override { return name; } bool useDefaultImplementationForNulls() const override { return false; } + /// See comment in ExecutableFunctionCapture. + bool useDefaultImplementationForNothing() const override { return false; } bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName &) const override { return return_type; } size_t getNumberOfArguments() const override { return capture->captured_types.size(); } diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h index 4aa76ceec28..68425ee496e 100644 --- a/src/Functions/FunctionsStringSearch.h +++ b/src/Functions/FunctionsStringSearch.h @@ -15,7 +15,6 @@ namespace DB { /** Search and replace functions in strings: - * * position(haystack, needle) - the normal search for a substring in a string, returns the position (in bytes) of the found substring starting with 1, or 0 if no substring is found. * positionUTF8(haystack, needle) - the same, but the position is calculated at code points, provided that the string is encoded in UTF-8. * positionCaseInsensitive(haystack, needle) @@ -24,13 +23,29 @@ namespace DB * like(haystack, pattern) - search by the regular expression LIKE; Returns 0 or 1. Case-insensitive, but only for Latin. * notLike(haystack, pattern) * + * ilike(haystack, pattern) - like 'like' but case-insensitive + * notIlike(haystack, pattern) + * * match(haystack, pattern) - search by regular expression re2; Returns 0 or 1. - * multiMatchAny(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- search by re2 regular expressions pattern_i; Returns 0 or 1 if any pattern_i matches. - * multiMatchAnyIndex(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- search by re2 regular expressions pattern_i; Returns index of any match or zero if none; - * multiMatchAllIndices(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- search by re2 regular expressions pattern_i; Returns an array of matched indices in any order; * * countSubstrings(haystack, needle) -- count number of occurrences of needle in haystack. * countSubstringsCaseInsensitive(haystack, needle) + * countSubstringsCaseInsensitiveUTF8(haystack, needle) + * + * hasToken() + * hasTokenCaseInsensitive() + * + * JSON stuff: + * visitParamExtractBool() + * simpleJSONExtractBool() + * visitParamExtractFloat() + * simpleJSONExtractFloat() + * visitParamExtractInt() + * simpleJSONExtractInt() + * visitParamExtractUInt() + * simpleJSONExtractUInt() + * visitParamHas() + * simpleJSONHas() * * Applies regexp re2 and pulls: * - the first subpattern, if the regexp has a subpattern; @@ -70,11 +85,7 @@ public: ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { - if (!Impl::use_default_implementation_for_constants) - return ColumnNumbers{}; - if (!Impl::supports_start_pos) - return ColumnNumbers{1, 2}; - return ColumnNumbers{1, 2, 3}; + return Impl::getArgumentsThatAreAlwaysConstant(); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -104,8 +115,6 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override { - using ResultType = typename Impl::ResultType; - const ColumnPtr & column_haystack = arguments[0].column; const ColumnPtr & column_needle = arguments[1].column; @@ -116,6 +125,8 @@ public: const ColumnConst * col_haystack_const = typeid_cast(&*column_haystack); const ColumnConst * col_needle_const = typeid_cast(&*column_needle); + using ResultType = typename Impl::ResultType; + if constexpr (!Impl::use_default_implementation_for_constants) { bool is_col_start_pos_const = column_start_pos == nullptr || isColumnConst(*column_start_pos); @@ -162,6 +173,14 @@ public: col_needle_const->getValue(), column_start_pos, vec_res); + else if (col_haystack_vector_fixed && col_needle_vector) + Impl::vectorFixedVector( + col_haystack_vector_fixed->getChars(), + col_haystack_vector_fixed->getN(), + col_needle_vector->getChars(), + col_needle_vector->getOffsets(), + column_start_pos, + vec_res); else if (col_haystack_vector_fixed && col_needle_const) Impl::vectorFixedConstant( col_haystack_vector_fixed->getChars(), diff --git a/src/Functions/FunctionsVisitParam.h b/src/Functions/FunctionsVisitParam.h index 09fcf8659ed..5f86923b0d1 100644 --- a/src/Functions/FunctionsVisitParam.h +++ b/src/Functions/FunctionsVisitParam.h @@ -83,10 +83,12 @@ struct ExtractParamImpl static constexpr bool supports_start_pos = false; static constexpr auto name = Name::name; + static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1, 2};} + /// It is assumed that `res` is the correct size and initialized with zeros. static void vectorConstant( - const ColumnString::Chars & data, - const ColumnString::Offsets & offsets, + const ColumnString::Chars & haystack_data, + const ColumnString::Offsets & haystack_offsets, std::string needle, const ColumnPtr & start_pos, PaddedPODArray & res) @@ -97,9 +99,9 @@ struct ExtractParamImpl /// We are looking for a parameter simply as a substring of the form "name" needle = "\"" + needle + "\":"; - const UInt8 * begin = data.data(); + const UInt8 * const begin = haystack_data.data(); + const UInt8 * const end = haystack_data.data() + haystack_data.size(); const UInt8 * pos = begin; - const UInt8 * end = pos + data.size(); /// The current index in the string array. size_t i = 0; @@ -110,19 +112,19 @@ struct ExtractParamImpl while (pos < end && end != (pos = searcher.search(pos, end - pos))) { /// Let's determine which index it belongs to. - while (begin + offsets[i] <= pos) + while (begin + haystack_offsets[i] <= pos) { res[i] = 0; ++i; } /// We check that the entry does not pass through the boundaries of strings. - if (pos + needle.size() < begin + offsets[i]) - res[i] = ParamExtractor::extract(pos + needle.size(), begin + offsets[i] - 1); /// don't include terminating zero + if (pos + needle.size() < begin + haystack_offsets[i]) + res[i] = ParamExtractor::extract(pos + needle.size(), begin + haystack_offsets[i] - 1); /// don't include terminating zero else res[i] = 0; - pos = begin + offsets[i]; + pos = begin + haystack_offsets[i]; ++i; } @@ -145,6 +147,12 @@ struct ExtractParamImpl { throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name); } + + template + static void vectorFixedVector(Args &&...) + { + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name); + } }; @@ -153,20 +161,20 @@ struct ExtractParamImpl template struct ExtractParamToStringImpl { - static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, + static void vector(const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, std::string needle, ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) { /// Constant 5 is taken from a function that performs a similar task FunctionsStringSearch.h::ExtractImpl - res_data.reserve(data.size() / 5); - res_offsets.resize(offsets.size()); + res_data.reserve(haystack_data.size() / 5); + res_offsets.resize(haystack_offsets.size()); /// We are looking for a parameter simply as a substring of the form "name" needle = "\"" + needle + "\":"; - const UInt8 * begin = data.data(); + const UInt8 * const begin = haystack_data.data(); + const UInt8 * const end = haystack_data.data() + haystack_data.size(); const UInt8 * pos = begin; - const UInt8 * end = pos + data.size(); /// The current index in the string array. size_t i = 0; @@ -177,7 +185,7 @@ struct ExtractParamToStringImpl while (pos < end && end != (pos = searcher.search(pos, end - pos))) { /// Determine which index it belongs to. - while (begin + offsets[i] <= pos) + while (begin + haystack_offsets[i] <= pos) { res_data.push_back(0); res_offsets[i] = res_data.size(); @@ -185,10 +193,10 @@ struct ExtractParamToStringImpl } /// We check that the entry does not pass through the boundaries of strings. - if (pos + needle.size() < begin + offsets[i]) - ParamExtractor::extract(pos + needle.size(), begin + offsets[i], res_data); + if (pos + needle.size() < begin + haystack_offsets[i]) + ParamExtractor::extract(pos + needle.size(), begin + haystack_offsets[i], res_data); - pos = begin + offsets[i]; + pos = begin + haystack_offsets[i]; res_data.push_back(0); res_offsets[i] = res_data.size(); diff --git a/src/Functions/GatherUtils/sliceHasImplAnyAll.h b/src/Functions/GatherUtils/sliceHasImplAnyAll.h index 68f31006b4f..21c80b742fd 100644 --- a/src/Functions/GatherUtils/sliceHasImplAnyAll.h +++ b/src/Functions/GatherUtils/sliceHasImplAnyAll.h @@ -4,18 +4,11 @@ #include "Slices.h" #include "sliceEqualElements.h" -#if defined(__SSE4_2__) - #include - #include - #include -#endif - -#if defined(__AVX2__) - #include -#endif - #include +#if USE_MULTITARGET_CODE +#include +#endif namespace DB::GatherUtils { @@ -67,15 +60,14 @@ inline ALWAYS_INLINE bool hasAllIntegralLoopRemainder( return true; } - -#if defined(__AVX2__) +#if USE_MULTITARGET_CODE DECLARE_AVX2_SPECIFIC_CODE ( // AVX2 Int64, UInt64 specialization template requires (std::is_same_v || std::is_same_v) -inline ALWAYS_INLINE bool sliceHasImplAnyAllImplInt64( +bool sliceHasImplAnyAllImplInt64( const NumericArraySlice & first, const NumericArraySlice & second, const UInt8 * first_null_map, @@ -165,7 +157,7 @@ inline ALWAYS_INLINE bool sliceHasImplAnyAllImplInt64( // AVX2 Int32, UInt32 specialization template requires (std::is_same_v || std::is_same_v) -inline ALWAYS_INLINE bool sliceHasImplAnyAllImplInt32( +bool sliceHasImplAnyAllImplInt32( const NumericArraySlice & first, const NumericArraySlice & second, const UInt8 * first_null_map, @@ -278,7 +270,7 @@ inline ALWAYS_INLINE bool sliceHasImplAnyAllImplInt32( // AVX2 Int16, UInt16 specialization template requires (std::is_same_v || std::is_same_v) -inline ALWAYS_INLINE bool sliceHasImplAnyAllImplInt16( +bool sliceHasImplAnyAllImplInt16( const NumericArraySlice & first, const NumericArraySlice & second, const UInt8 * first_null_map, @@ -422,10 +414,6 @@ inline ALWAYS_INLINE bool sliceHasImplAnyAllImplInt16( ) -#endif - -#if defined(__SSE4_2__) - DECLARE_SSE42_SPECIFIC_CODE ( // SSE4.2 Int64, UInt64 specialization @@ -895,8 +883,6 @@ inline ALWAYS_INLINE bool sliceHasImplAnyAll(const FirstSliceType & first, const #if USE_MULTITARGET_CODE if constexpr (search_type == ArraySearchType::All && std::is_same_v) { - -#if defined(__AVX2__) if (isArchSupported(TargetArch::AVX2)) { if constexpr (std::is_same_v> || std::is_same_v>) @@ -912,13 +898,12 @@ inline ALWAYS_INLINE bool sliceHasImplAnyAll(const FirstSliceType & first, const return GatherUtils::TargetSpecific::AVX2::sliceHasImplAnyAllImplInt64(first, second, first_null_map, second_null_map); } } -#endif if (isArchSupported(TargetArch::SSE42)) { if constexpr (std::is_same_v> || std::is_same_v>) { - return TargetSpecific::SSE42::sliceHasImplAnyAllImplInt8(first, second, first_null_map, second_null_map); + return GatherUtils::TargetSpecific::SSE42::sliceHasImplAnyAllImplInt8(first, second, first_null_map, second_null_map); } else if constexpr (std::is_same_v> || std::is_same_v>) { diff --git a/src/Functions/HasTokenImpl.h b/src/Functions/HasTokenImpl.h index ec33a07fce3..9328bd99139 100644 --- a/src/Functions/HasTokenImpl.h +++ b/src/Functions/HasTokenImpl.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -14,7 +15,7 @@ namespace ErrorCodes /** Token search the string, means that needle must be surrounded by some separator chars, like whitespace or puctuation. */ -template +template struct HasTokenImpl { using ResultType = UInt8; @@ -23,9 +24,11 @@ struct HasTokenImpl static constexpr bool supports_start_pos = false; static constexpr auto name = Name::name; + static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1, 2};} + static void vectorConstant( - const ColumnString::Chars & data, - const ColumnString::Offsets & offsets, + const ColumnString::Chars & haystack_data, + const ColumnString::Offsets & haystack_offsets, const std::string & pattern, const ColumnPtr & start_pos, PaddedPODArray & res) @@ -33,12 +36,12 @@ struct HasTokenImpl if (start_pos != nullptr) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' does not support start_pos argument", name); - if (offsets.empty()) + if (haystack_offsets.empty()) return; - const UInt8 * begin = data.data(); + const UInt8 * const begin = haystack_data.data(); + const UInt8 * const end = haystack_data.data() + haystack_data.size(); const UInt8 * pos = begin; - const UInt8 * end = pos + data.size(); /// The current index in the array of strings. size_t i = 0; @@ -49,25 +52,25 @@ struct HasTokenImpl while (pos < end && end != (pos = searcher.search(pos, end - pos))) { /// Let's determine which index it refers to. - while (begin + offsets[i] <= pos) + while (begin + haystack_offsets[i] <= pos) { - res[i] = negate_result; + res[i] = negate; ++i; } /// We check that the entry does not pass through the boundaries of strings. - if (pos + pattern.size() < begin + offsets[i]) - res[i] = !negate_result; + if (pos + pattern.size() < begin + haystack_offsets[i]) + res[i] = !negate; else - res[i] = negate_result; + res[i] = negate; - pos = begin + offsets[i]; + pos = begin + haystack_offsets[i]; ++i; } /// Tail, in which there can be no substring. if (i < res.size()) - memset(&res[i], negate_result, (res.size() - i) * sizeof(res[0])); + memset(&res[i], negate, (res.size() - i) * sizeof(res[0])); } template @@ -88,6 +91,12 @@ struct HasTokenImpl { throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name); } + + template + static void vectorFixedVector(Args &&...) + { + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name); + } }; } diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp index 19638c78daf..171a6f55a39 100644 --- a/src/Functions/IFunction.cpp +++ b/src/Functions/IFunction.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -203,6 +204,32 @@ ColumnPtr IExecutableFunction::defaultImplementationForNulls( return nullptr; } +ColumnPtr IExecutableFunction::defaultImplementationForNothing( + const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count) const +{ + if (!useDefaultImplementationForNothing()) + return nullptr; + + bool is_nothing_type_presented = false; + for (const auto & arg : args) + is_nothing_type_presented |= isNothing(arg.type); + + if (!is_nothing_type_presented) + return nullptr; + + if (!isNothing(result_type)) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Function {} with argument with type Nothing and default implementation for Nothing " + "is expected to return result with type Nothing, got {}", + getName(), + result_type->getName()); + + if (input_rows_count > 0) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot create non-empty column with type Nothing"); + return ColumnNothing::create(0); +} + ColumnPtr IExecutableFunction::executeWithoutLowCardinalityColumns( const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run) const { @@ -212,6 +239,9 @@ ColumnPtr IExecutableFunction::executeWithoutLowCardinalityColumns( if (auto res = defaultImplementationForNulls(args, result_type, input_rows_count, dry_run)) return res; + if (auto res = defaultImplementationForNothing(args, result_type, input_rows_count)) + return res; + ColumnPtr res; if (dry_run) res = executeDryRunImpl(args, result_type, input_rows_count); @@ -250,16 +280,22 @@ ColumnPtr IExecutableFunction::executeWithoutSparseColumns(const ColumnsWithType : columns_without_low_cardinality.front().column->size(); auto res = executeWithoutLowCardinalityColumns(columns_without_low_cardinality, dictionary_type, new_input_rows_count, dry_run); - auto keys = res->convertToFullColumnIfConst(); + bool res_is_constant = isColumnConst(*res); + auto keys = res_is_constant + ? res->cloneResized(1)->convertToFullColumnIfConst() + : res; auto res_mut_dictionary = DataTypeLowCardinality::createColumnUnique(*res_low_cardinality_type->getDictionaryType()); ColumnPtr res_indexes = res_mut_dictionary->uniqueInsertRangeFrom(*keys, 0, keys->size()); ColumnUniquePtr res_dictionary = std::move(res_mut_dictionary); - if (indexes) + if (indexes && !res_is_constant) result = ColumnLowCardinality::create(res_dictionary, res_indexes->index(*indexes, 0)); else result = ColumnLowCardinality::create(res_dictionary, res_indexes); + + if (res_is_constant) + result = ColumnConst::create(std::move(result), input_rows_count); } else { @@ -430,6 +466,15 @@ DataTypePtr IFunctionOverloadResolver::getReturnTypeWithoutLowCardinality(const } } + if (!arguments.empty() && useDefaultImplementationForNothing()) + { + for (const auto & arg : arguments) + { + if (isNothing(arg.type)) + return std::make_shared(); + } + } + return getReturnTypeImpl(arguments); } diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 7b272fef53d..95af8a61aae 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -63,6 +63,11 @@ protected: */ virtual bool useDefaultImplementationForNulls() const { return true; } + /** Default implementation in presence of arguments with type Nothing is the following: + * If some of arguments have type Nothing then default implementation is to return constant column with type Nothing + */ + virtual bool useDefaultImplementationForNothing() const { return true; } + /** If the function have non-zero number of arguments, * and if all arguments are constant, that we could automatically provide default implementation: * arguments are converted to ordinary columns with single value, then function is executed as usual, @@ -100,6 +105,9 @@ private: ColumnPtr defaultImplementationForNulls( const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run) const; + ColumnPtr defaultImplementationForNothing( + const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count) const; + ColumnPtr executeWithoutLowCardinalityColumns( const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run) const; @@ -166,8 +174,8 @@ public: /** If function isSuitableForConstantFolding then, this method will be called during query analyzis * if some arguments are constants. For example logical functions (AndFunction, OrFunction) can * return they result based on some constant arguments. - * Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForConstants, - * useDefaultImplementationForLowCardinality are not applied. + * Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForNothing, + * useDefaultImplementationForConstants, useDefaultImplementationForLowCardinality are not applied. */ virtual ColumnPtr getConstantResultForNonConstArguments( const ColumnsWithTypeAndName & /* arguments */, const DataTypePtr & /* result_type */) const { return nullptr; } @@ -354,7 +362,13 @@ protected: */ virtual bool useDefaultImplementationForNulls() const { return true; } - /** If useDefaultImplementationForNulls() is true, then change arguments for getReturnType() and build(). + /** If useDefaultImplementationForNothing() is true, then change arguments for getReturnType() and build(): + * if some of arguments are Nothing then don't call getReturnType(), call build() with return_type = Nothing, + * Otherwise build returns build(arguments, getReturnType(arguments)); + */ + virtual bool useDefaultImplementationForNothing() const { return true; } + + /** If useDefaultImplementationForLowCardinalityColumns() is true, then change arguments for getReturnType() and build(). * If function arguments has low cardinality types, convert them to ordinary types. * getReturnType returns ColumnLowCardinality if at least one argument type is ColumnLowCardinality. */ @@ -403,6 +417,11 @@ public: */ virtual bool useDefaultImplementationForNulls() const { return true; } + /** Default implementation in presence of arguments with type Nothing is the following: + * If some of arguments have type Nothing then default implementation is to return constant column with type Nothing + */ + virtual bool useDefaultImplementationForNothing() const { return true; } + /** If the function have non-zero number of arguments, * and if all arguments are constant, that we could automatically provide default implementation: * arguments are converted to ordinary columns with single value, then function is executed as usual, diff --git a/src/Functions/IFunctionAdaptors.h b/src/Functions/IFunctionAdaptors.h index ec43087ad66..dfb90fd3975 100644 --- a/src/Functions/IFunctionAdaptors.h +++ b/src/Functions/IFunctionAdaptors.h @@ -27,6 +27,7 @@ protected: } bool useDefaultImplementationForNulls() const final { return function->useDefaultImplementationForNulls(); } + bool useDefaultImplementationForNothing() const final { return function->useDefaultImplementationForNothing(); } bool useDefaultImplementationForConstants() const final { return function->useDefaultImplementationForConstants(); } bool useDefaultImplementationForLowCardinalityColumns() const final { return function->useDefaultImplementationForLowCardinalityColumns(); } bool useDefaultImplementationForSparseColumns() const final { return function->useDefaultImplementationForSparseColumns(); } @@ -124,6 +125,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { return function->getReturnTypeImpl(arguments); } bool useDefaultImplementationForNulls() const override { return function->useDefaultImplementationForNulls(); } + bool useDefaultImplementationForNothing() const override { return function->useDefaultImplementationForNothing(); } bool useDefaultImplementationForLowCardinalityColumns() const override { return function->useDefaultImplementationForLowCardinalityColumns(); } bool useDefaultImplementationForSparseColumns() const override { return function->useDefaultImplementationForSparseColumns(); } bool canBeExecutedOnLowCardinalityDictionary() const override { return function->canBeExecutedOnLowCardinalityDictionary(); } diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h index 026b38b997b..54aaa3116fd 100644 --- a/src/Functions/MatchImpl.h +++ b/src/Functions/MatchImpl.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "Regexps.h" #include "config_functions.h" @@ -17,24 +18,24 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_COLUMN; + extern const int LOGICAL_ERROR; extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +namespace impl +{ /// Is the [I]LIKE expression reduced to finding a substring in a string? -static inline bool likePatternIsStrstr(const String & pattern, String & res) +inline bool likePatternIsSubstring(std::string_view pattern, String & res) { if (pattern.size() < 2 || pattern.front() != '%' || pattern.back() != '%') return false; - res = ""; + res.clear(); res.reserve(pattern.size() - 2); - const char * pos = pattern.data(); - const char * end = pos + pattern.size(); - - ++pos; - --end; + const char * pos = pattern.data() + 1; + const char * const end = pattern.data() + pattern.size() - 1; while (pos < end) { @@ -60,17 +61,24 @@ static inline bool likePatternIsStrstr(const String & pattern, String & res) return true; } -/** 'like' - if true, treat pattern as SQL LIKE or ILIKE; if false - treat pattern as re2 regexp. +} + +/** 'like' - if true, treat pattern as SQL LIKE, otherwise as re2 regexp. + * 'negate' - if true, negate result + * 'case_insensitive' - if true, match case insensitively + * * NOTE: We want to run regexp search for whole columns by one call (as implemented in function 'position') * but for that, regexp engine must support \0 bytes and their interpretation as string boundaries. */ -template +template struct MatchImpl { static constexpr bool use_default_implementation_for_constants = true; static constexpr bool supports_start_pos = false; static constexpr auto name = Name::name; + static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {2};} + using ResultType = UInt8; using Searcher = std::conditional_t; static void vectorConstant( - const ColumnString::Chars & data, - const ColumnString::Offsets & offsets, - const String & pattern, - const ColumnPtr & start_pos, + const ColumnString::Chars & haystack_data, + const ColumnString::Offsets & haystack_offsets, + const String & needle, + const ColumnPtr & start_pos_, PaddedPODArray & res) { - if (start_pos != nullptr) + if (start_pos_ != nullptr) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name); - if (offsets.empty()) + if (haystack_offsets.empty()) return; /// A simple case where the [I]LIKE expression reduces to finding a substring in a string String strstr_pattern; - if (like && likePatternIsStrstr(pattern, strstr_pattern)) + if (like && impl::likePatternIsSubstring(needle, strstr_pattern)) { - const UInt8 * const begin = data.data(); - const UInt8 * const end = data.data() + data.size(); + const UInt8 * const begin = haystack_data.data(); + const UInt8 * const end = haystack_data.data() + haystack_data.size(); const UInt8 * pos = begin; /// The current index in the array of strings. @@ -109,31 +117,29 @@ struct MatchImpl while (pos < end && end != (pos = searcher.search(pos, end - pos))) { /// Let's determine which index it refers to. - while (begin + offsets[i] <= pos) + while (begin + haystack_offsets[i] <= pos) { - res[i] = revert; + res[i] = negate; ++i; } /// We check that the entry does not pass through the boundaries of strings. - if (pos + strstr_pattern.size() < begin + offsets[i]) - res[i] = !revert; + if (pos + strstr_pattern.size() < begin + haystack_offsets[i]) + res[i] = !negate; else - res[i] = revert; + res[i] = negate; - pos = begin + offsets[i]; + pos = begin + haystack_offsets[i]; ++i; } /// Tail, in which there can be no substring. if (i < res.size()) - memset(&res[i], revert, (res.size() - i) * sizeof(res[0])); + memset(&res[i], negate, (res.size() - i) * sizeof(res[0])); } else { - size_t size = offsets.size(); - - auto regexp = Regexps::get(pattern); + auto regexp = Regexps::get(needle); String required_substring; bool is_trivial; @@ -141,37 +147,39 @@ struct MatchImpl regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix); + size_t haystack_size = haystack_offsets.size(); + if (required_substring.empty()) { if (!regexp->getRE2()) /// An empty regexp. Always matches. { - if (size) - memset(res.data(), 1, size * sizeof(res[0])); + if (haystack_size) + memset(res.data(), 1, haystack_size * sizeof(res[0])); } else { size_t prev_offset = 0; - for (size_t i = 0; i < size; ++i) + for (size_t i = 0; i < haystack_size; ++i) { - res[i] = revert + res[i] = negate ^ regexp->getRE2()->Match( - re2_st::StringPiece(reinterpret_cast(&data[prev_offset]), offsets[i] - prev_offset - 1), + {reinterpret_cast(&haystack_data[prev_offset]), haystack_offsets[i] - prev_offset - 1}, 0, - offsets[i] - prev_offset - 1, + haystack_offsets[i] - prev_offset - 1, re2_st::RE2::UNANCHORED, nullptr, 0); - prev_offset = offsets[i]; + prev_offset = haystack_offsets[i]; } } } else { - /// NOTE This almost matches with the case of LikePatternIsStrstr. + /// NOTE This almost matches with the case of impl::likePatternIsSubstring. - const UInt8 * const begin = data.data(); - const UInt8 * const end = data.begin() + data.size(); + const UInt8 * const begin = haystack_data.data(); + const UInt8 * const end = haystack_data.begin() + haystack_data.size(); const UInt8 * pos = begin; /// The current index in the array of strings. @@ -183,79 +191,78 @@ struct MatchImpl while (pos < end && end != (pos = searcher.search(pos, end - pos))) { /// Determine which index it refers to. - while (begin + offsets[i] <= pos) + while (begin + haystack_offsets[i] <= pos) { - res[i] = revert; + res[i] = negate; ++i; } /// We check that the entry does not pass through the boundaries of strings. - if (pos + required_substring.size() < begin + offsets[i]) + if (pos + required_substring.size() < begin + haystack_offsets[i]) { /// And if it does not, if necessary, we check the regexp. if (is_trivial) - res[i] = !revert; + res[i] = !negate; else { - const char * str_data = reinterpret_cast(&data[offsets[i - 1]]); - size_t str_size = offsets[i] - offsets[i - 1] - 1; + const char * str_data = reinterpret_cast(&haystack_data[haystack_offsets[i - 1]]); + size_t str_size = haystack_offsets[i] - haystack_offsets[i - 1] - 1; /** Even in the case of `required_substring_is_prefix` use UNANCHORED check for regexp, * so that it can match when `required_substring` occurs into the string several times, * and at the first occurrence, the regexp is not a match. */ + const size_t start_pos = (required_substring_is_prefix) ? (reinterpret_cast(pos) - str_data) : 0; + const size_t end_pos = str_size; - if (required_substring_is_prefix) - res[i] = revert - ^ regexp->getRE2()->Match( - re2_st::StringPiece(str_data, str_size), - reinterpret_cast(pos) - str_data, - str_size, - re2_st::RE2::UNANCHORED, - nullptr, - 0); - else - res[i] = revert - ^ regexp->getRE2()->Match( - re2_st::StringPiece(str_data, str_size), 0, str_size, re2_st::RE2::UNANCHORED, nullptr, 0); + res[i] = negate + ^ regexp->getRE2()->Match( + {str_data, str_size}, + start_pos, + end_pos, + re2_st::RE2::UNANCHORED, + nullptr, + 0); } } else - res[i] = revert; + res[i] = negate; - pos = begin + offsets[i]; + pos = begin + haystack_offsets[i]; ++i; } /// Tail, in which there can be no substring. if (i < res.size()) - memset(&res[i], revert, (res.size() - i) * sizeof(res[0])); + memset(&res[i], negate, (res.size() - i) * sizeof(res[0])); } } } /// Very carefully crafted copy-paste. static void vectorFixedConstant( - const ColumnString::Chars & data, size_t n, const String & pattern, + const ColumnString::Chars & haystack, + size_t N, + const String & needle, PaddedPODArray & res) { - if (data.empty()) + if (haystack.empty()) return; /// A simple case where the LIKE expression reduces to finding a substring in a string String strstr_pattern; - if (like && likePatternIsStrstr(pattern, strstr_pattern)) + if (like && impl::likePatternIsSubstring(needle, strstr_pattern)) { - const UInt8 * begin = data.data(); + const UInt8 * const begin = haystack.data(); + const UInt8 * const end = haystack.data() + haystack.size(); const UInt8 * pos = begin; - const UInt8 * end = pos + data.size(); size_t i = 0; const UInt8 * next_pos = begin; - /// If pattern is larger than string size - it cannot be found. - if (strstr_pattern.size() <= n) + /// If needle is larger than string size - it cannot be found. + if (strstr_pattern.size() <= N) { Searcher searcher(strstr_pattern.data(), strstr_pattern.size(), end - pos); @@ -263,19 +270,19 @@ struct MatchImpl while (pos < end && end != (pos = searcher.search(pos, end - pos))) { /// Let's determine which index it refers to. - while (next_pos + n <= pos) + while (next_pos + N <= pos) { - res[i] = revert; - next_pos += n; + res[i] = negate; + next_pos += N; ++i; } - next_pos += n; + next_pos += N; /// We check that the entry does not pass through the boundaries of strings. if (pos + strstr_pattern.size() <= next_pos) - res[i] = !revert; + res[i] = !negate; else - res[i] = revert; + res[i] = negate; pos = next_pos; ++i; @@ -284,13 +291,11 @@ struct MatchImpl /// Tail, in which there can be no substring. if (i < res.size()) - memset(&res[i], revert, (res.size() - i) * sizeof(res[0])); + memset(&res[i], negate, (res.size() - i) * sizeof(res[0])); } else { - size_t size = data.size() / n; - - auto regexp = Regexps::get(pattern); + auto regexp = Regexps::get(needle); String required_substring; bool is_trivial; @@ -298,44 +303,46 @@ struct MatchImpl regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix); + const size_t haystack_size = haystack.size() / N; + if (required_substring.empty()) { if (!regexp->getRE2()) /// An empty regexp. Always matches. { - if (size) - memset(res.data(), 1, size * sizeof(res[0])); + if (haystack_size) + memset(res.data(), 1, haystack_size * sizeof(res[0])); } else { size_t offset = 0; - for (size_t i = 0; i < size; ++i) + for (size_t i = 0; i < haystack_size; ++i) { - res[i] = revert + res[i] = negate ^ regexp->getRE2()->Match( - re2_st::StringPiece(reinterpret_cast(&data[offset]), n), + {reinterpret_cast(&haystack[offset]), N}, 0, - n, + N, re2_st::RE2::UNANCHORED, nullptr, 0); - offset += n; + offset += N; } } } else { - /// NOTE This almost matches with the case of LikePatternIsStrstr. + /// NOTE This almost matches with the case of likePatternIsSubstring. - const UInt8 * begin = data.data(); + const UInt8 * const begin = haystack.data(); + const UInt8 * const end = haystack.data() + haystack.size(); const UInt8 * pos = begin; - const UInt8 * end = pos + data.size(); size_t i = 0; const UInt8 * next_pos = begin; /// If required substring is larger than string size - it cannot be found. - if (required_substring.size() <= n) + if (required_substring.size() <= N) { Searcher searcher(required_substring.data(), required_substring.size(), end - pos); @@ -343,46 +350,43 @@ struct MatchImpl while (pos < end && end != (pos = searcher.search(pos, end - pos))) { /// Let's determine which index it refers to. - while (next_pos + n <= pos) + while (next_pos + N <= pos) { - res[i] = revert; - next_pos += n; + res[i] = negate; + next_pos += N; ++i; } - next_pos += n; + next_pos += N; if (pos + required_substring.size() <= next_pos) { /// And if it does not, if necessary, we check the regexp. if (is_trivial) - res[i] = !revert; + res[i] = !negate; else { - const char * str_data = reinterpret_cast(next_pos - n); + const char * str_data = reinterpret_cast(next_pos - N); /** Even in the case of `required_substring_is_prefix` use UNANCHORED check for regexp, * so that it can match when `required_substring` occurs into the string several times, * and at the first occurrence, the regexp is not a match. */ + const size_t start_pos = (required_substring_is_prefix) ? (reinterpret_cast(pos) - str_data) : 0; + const size_t end_pos = N; - if (required_substring_is_prefix) - res[i] = revert - ^ regexp->getRE2()->Match( - re2_st::StringPiece(str_data, n), - reinterpret_cast(pos) - str_data, - n, - re2_st::RE2::UNANCHORED, - nullptr, - 0); - else - res[i] = revert - ^ regexp->getRE2()->Match( - re2_st::StringPiece(str_data, n), 0, n, re2_st::RE2::UNANCHORED, nullptr, 0); + res[i] = negate + ^ regexp->getRE2()->Match( + {str_data, N}, + start_pos, + end_pos, + re2_st::RE2::UNANCHORED, + nullptr, + 0); } } else - res[i] = revert; + res[i] = negate; pos = next_pos; ++i; @@ -391,22 +395,248 @@ struct MatchImpl /// Tail, in which there can be no substring. if (i < res.size()) - memset(&res[i], revert, (res.size() - i) * sizeof(res[0])); + memset(&res[i], negate, (res.size() - i) * sizeof(res[0])); } } } - template - static void vectorVector(Args &&...) + static void vectorVector( + const ColumnString::Chars & haystack_data, + const ColumnString::Offsets & haystack_offsets, + const ColumnString::Chars & needle_data, + const ColumnString::Offsets & needle_offset, + const ColumnPtr & start_pos_, + PaddedPODArray & res) { - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support non-constant needle argument", name); + const size_t haystack_size = haystack_offsets.size(); + + if (haystack_size != needle_offset.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Function '{}' unexpectedly received a different number of haystacks and needles", name); + + if (start_pos_ != nullptr) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Function '{}' doesn't support start_pos argument", name); + + if (haystack_offsets.empty()) + return; + + String required_substr; + bool is_trivial; + bool required_substring_is_prefix; /// for `anchored` execution of the regexp. + + size_t prev_haystack_offset = 0; + size_t prev_needle_offset = 0; + + for (size_t i = 0; i < haystack_size; ++i) + { + const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset]; + const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1; + + const auto * const cur_needle_data = &needle_data[prev_needle_offset]; + const size_t cur_needle_length = needle_offset[i] - prev_needle_offset - 1; + + const auto & needle = String( + reinterpret_cast(cur_needle_data), + cur_needle_length); + + if (like && impl::likePatternIsSubstring(needle, required_substr)) + { + if (required_substr.size() > cur_haystack_length) + res[i] = negate; + else + { + Searcher searcher(required_substr.data(), required_substr.size(), cur_haystack_length); + const auto * match = searcher.search(cur_haystack_data, cur_haystack_length); + res[i] = negate + ^ (match != cur_haystack_data + cur_haystack_length); + } + } + else + { + // each row is expected to contain a different like/re2 pattern + // --> bypass the regexp cache, instead construct the pattern on-the-fly + const int flags = Regexps::buildRe2Flags(); + const auto & regexp = Regexps::Regexp(Regexps::createRegexp(needle, flags)); + + regexp.getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix); + + if (required_substr.empty()) + { + if (!regexp.getRE2()) /// An empty regexp. Always matches. + { + res[i] = 1; + } + else + { + res[i] = negate + ^ regexp.getRE2()->Match( + {reinterpret_cast(cur_haystack_data), cur_haystack_length}, + 0, + cur_haystack_length, + re2_st::RE2::UNANCHORED, + nullptr, + 0); + } + } + else + { + Searcher searcher(required_substr.data(), required_substr.size(), cur_haystack_length); + const auto * match = searcher.search(cur_haystack_data, cur_haystack_length); + + if (match == cur_haystack_data + cur_haystack_length) + { + res[i] = negate; // no match + } + else + { + if (is_trivial) + { + res[i] = !negate; // no wildcards in pattern + } + else + { + const size_t start_pos = (required_substring_is_prefix) ? (match - cur_haystack_data) : 0; + const size_t end_pos = cur_haystack_length; + + res[i] = negate + ^ regexp.getRE2()->Match( + {reinterpret_cast(cur_haystack_data), cur_haystack_length}, + start_pos, + end_pos, + re2_st::RE2::UNANCHORED, + nullptr, + 0); + } + } + } + } + + prev_haystack_offset = haystack_offsets[i]; + prev_needle_offset = needle_offset[i]; + } + } + + static void vectorFixedVector( + const ColumnString::Chars & haystack, + size_t N, + const ColumnString::Chars & needle_data, + const ColumnString::Offsets & needle_offset, + const ColumnPtr & start_pos_, + PaddedPODArray & res) + { + const size_t haystack_size = haystack.size()/N; + + if (haystack_size != needle_offset.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Function '{}' unexpectedly received a different number of haystacks and needles", name); + + if (start_pos_ != nullptr) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Function '{}' doesn't support start_pos argument", name); + + if (haystack.empty()) + return; + + String required_substr; + bool is_trivial; + bool required_substring_is_prefix; // for `anchored` execution of the regexp. + + size_t prev_haystack_offset = 0; + size_t prev_needle_offset = 0; + + for (size_t i = 0; i < haystack_size; ++i) + { + const auto * const cur_haystack_data = &haystack[prev_haystack_offset]; + const size_t cur_haystack_length = N; + + const auto * const cur_needle_data = &needle_data[prev_needle_offset]; + const size_t cur_needle_length = needle_offset[i] - prev_needle_offset - 1; + + const auto & needle = String( + reinterpret_cast(cur_needle_data), + cur_needle_length); + + if (like && impl::likePatternIsSubstring(needle, required_substr)) + { + if (required_substr.size() > cur_haystack_length) + res[i] = negate; + else + { + Searcher searcher(required_substr.data(), required_substr.size(), cur_haystack_length); + const auto * match = searcher.search(cur_haystack_data, cur_haystack_length); + res[i] = negate + ^ (match != cur_haystack_data + cur_haystack_length); + } + } + else + { + // each row is expected to contain a different like/re2 pattern + // --> bypass the regexp cache, instead construct the pattern on-the-fly + const int flags = Regexps::buildRe2Flags(); + const auto & regexp = Regexps::Regexp(Regexps::createRegexp(needle, flags)); + + regexp.getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix); + + if (required_substr.empty()) + { + if (!regexp.getRE2()) /// An empty regexp. Always matches. + { + res[i] = 1; + } + else + { + res[i] = negate + ^ regexp.getRE2()->Match( + {reinterpret_cast(cur_haystack_data), cur_haystack_length}, + 0, + cur_haystack_length, + re2_st::RE2::UNANCHORED, + nullptr, + 0); + } + } + else + { + Searcher searcher(required_substr.data(), required_substr.size(), cur_haystack_length); + const auto * match = searcher.search(cur_haystack_data, cur_haystack_length); + + if (match == cur_haystack_data + cur_haystack_length) + { + res[i] = negate; // no match + } + else + { + if (is_trivial) + { + res[i] = !negate; // no wildcards in pattern + } + else + { + const size_t start_pos = (required_substring_is_prefix) ? (match - cur_haystack_data) : 0; + const size_t end_pos = cur_haystack_length; + + res[i] = negate + ^ regexp.getRE2()->Match( + {reinterpret_cast(cur_haystack_data), cur_haystack_length}, + start_pos, + end_pos, + re2_st::RE2::UNANCHORED, + nullptr, + 0); + } + } + } + } + prev_haystack_offset += N; + prev_needle_offset = needle_offset[i]; + } } - /// Search different needles in single haystack. template static void constantVector(Args &&...) { - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support non-constant needle argument", name); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support search with non-constant needles in constant haystack", name); } }; diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index f3e67008707..adf9e9b585f 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -11,8 +11,6 @@ #if USE_HYPERSCAN # include -#else -# include "MatchImpl.h" #endif diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h index 747c0e5e62c..595a3c8de5b 100644 --- a/src/Functions/MultiMatchAnyImpl.h +++ b/src/Functions/MultiMatchAnyImpl.h @@ -120,7 +120,7 @@ struct MultiMatchAnyImpl memset(accum.data(), 0, accum.size()); for (size_t j = 0; j < needles.size(); ++j) { - MatchImpl::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), nullptr, accum); + MatchImpl::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), nullptr, accum); for (size_t i = 0; i < res.size(); ++i) { if constexpr (FindAny) diff --git a/src/Functions/PositionImpl.h b/src/Functions/PositionImpl.h index d3b6d74c3cd..82e58cdc643 100644 --- a/src/Functions/PositionImpl.h +++ b/src/Functions/PositionImpl.h @@ -182,19 +182,21 @@ struct PositionImpl static constexpr bool supports_start_pos = true; static constexpr auto name = Name::name; + static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {};} + using ResultType = UInt64; /// Find one substring in many strings. static void vectorConstant( - const ColumnString::Chars & data, - const ColumnString::Offsets & offsets, + const ColumnString::Chars & haystack_data, + const ColumnString::Offsets & haystack_offsets, const std::string & needle, const ColumnPtr & start_pos, PaddedPODArray & res) { - const UInt8 * begin = data.data(); + const UInt8 * const begin = haystack_data.data(); + const UInt8 * const end = haystack_data.data() + haystack_data.size(); const UInt8 * pos = begin; - const UInt8 * end = pos + data.size(); /// Current index in the array of strings. size_t i = 0; @@ -205,7 +207,7 @@ struct PositionImpl while (pos < end && end != (pos = searcher.search(pos, end - pos))) { /// Determine which index it refers to. - while (begin + offsets[i] <= pos) + while (begin + haystack_offsets[i] <= pos) { res[i] = 0; ++i; @@ -213,14 +215,14 @@ struct PositionImpl auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0; /// We check that the entry does not pass through the boundaries of strings. - if (pos + needle.size() < begin + offsets[i]) + if (pos + needle.size() < begin + haystack_offsets[i]) { - auto res_pos = 1 + Impl::countChars(reinterpret_cast(begin + offsets[i - 1]), reinterpret_cast(pos)); + auto res_pos = 1 + Impl::countChars(reinterpret_cast(begin + haystack_offsets[i - 1]), reinterpret_cast(pos)); if (res_pos < start) { pos = reinterpret_cast(Impl::advancePos( reinterpret_cast(pos), - reinterpret_cast(begin + offsets[i]), + reinterpret_cast(begin + haystack_offsets[i]), start - res_pos)); continue; } @@ -230,7 +232,7 @@ struct PositionImpl { res[i] = 0; } - pos = begin + offsets[i]; + pos = begin + haystack_offsets[i]; ++i; } @@ -411,6 +413,12 @@ struct PositionImpl { throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name); } + + template + static void vectorFixedVector(Args &&...) + { + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name); + } }; } diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index 9a1938a3f32..dc94b75211c 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -38,7 +38,7 @@ namespace ErrorCodes namespace Regexps { - using Regexp = OptimizedRegularExpressionImpl; + using Regexp = OptimizedRegularExpressionSingleThreaded; using Pool = ObjectPoolMap; template @@ -50,6 +50,17 @@ namespace Regexps return {pattern, flags}; } + template + inline int buildRe2Flags() + { + int flags = OptimizedRegularExpression::RE_DOT_NL; + if constexpr (no_capture) + flags |= OptimizedRegularExpression::RE_NO_CAPTURE; + if constexpr (case_insensitive) + flags |= OptimizedRegularExpression::RE_CASELESS; + return flags; + } + /** Returns holder of an object from Pool. * You must hold the ownership while using the object. * In destructor, it returns the object back to the Pool for further reuse. @@ -62,14 +73,7 @@ namespace Regexps return known_regexps.get(pattern, [&pattern] { - int flags = OptimizedRegularExpression::RE_DOT_NL; - - if (no_capture) - flags |= OptimizedRegularExpression::RE_NO_CAPTURE; - - if (case_insensitive) - flags |= Regexps::Regexp::RE_CASELESS; - + const int flags = buildRe2Flags(); ProfileEvents::increment(ProfileEvents::RegexpCreated); return new Regexp{createRegexp(pattern, flags)}; }); diff --git a/src/Functions/array/CMakeLists.txt b/src/Functions/array/CMakeLists.txt index c98f4430078..9762674d6e9 100644 --- a/src/Functions/array/CMakeLists.txt +++ b/src/Functions/array/CMakeLists.txt @@ -1,7 +1,7 @@ include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_functions_array .) add_library(clickhouse_functions_array ${clickhouse_functions_array_sources} ${clickhouse_functions_array_headers}) -target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils ch_contrib::eigen) +target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_array PRIVATE "-g0") diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 58e6db86f75..0af68910b70 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -16,11 +17,13 @@ #include #include #include +#include #include #include #include +#include #include @@ -156,7 +159,7 @@ public: DataTypePtr nested_type = data_type->getNestedType(); - if (Impl::needBoolean() && !WhichDataType(nested_type).isUInt8()) + if (Impl::needBoolean() && !isUInt8(nested_type)) throw Exception("The only argument for function " + getName() + " must be array of UInt8. Found " + arguments[0].type->getName() + " instead", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -180,8 +183,14 @@ public: /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. DataTypePtr return_type = removeLowCardinality(data_type_function->getReturnType()); - if (Impl::needBoolean() && !WhichDataType(return_type).isUInt8()) - throw Exception("Expression for function " + getName() + " must return UInt8, found " + + /// Special cases when we need boolean lambda result: + /// - lambda may return Nullable(UInt8) column, in this case after lambda execution we will + /// replace all NULLs with 0 and return nested UInt8 column. + /// - lambda may return Nothing or Nullable(Nothing) because of default implementation of functions + /// for these types. In this case we will just create UInt8 const column full of 0. + if (Impl::needBoolean() && !isUInt8(removeNullable(return_type)) && !isNothing(removeNullable(return_type))) + throw Exception("Expression for function " + getName() + " must return UInt8 or Nullable(UInt8), found " + return_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); static_assert( @@ -316,11 +325,37 @@ public: auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); replicated_column_function->appendArguments(arrays); - auto lambda_result = replicated_column_function->reduce().column; - if (lambda_result->lowCardinality()) - lambda_result = lambda_result->convertToFullColumnIfLowCardinality(); + auto lambda_result = replicated_column_function->reduce(); + if (lambda_result.column->lowCardinality()) + lambda_result.column = lambda_result.column->convertToFullColumnIfLowCardinality(); - return Impl::execute(*column_first_array, lambda_result); + if (Impl::needBoolean()) + { + /// If result column is Nothing or Nullable(Nothing), just create const UInt8 column with 0 value. + if (isNothing(removeNullable(lambda_result.type))) + { + auto result_type = std::make_shared(); + lambda_result.column = result_type->createColumnConst(lambda_result.column->size(), 0); + } + /// If result column is Nullable(UInt8), then extract nested column and write 0 in all rows + /// when we have NULL. + else if (lambda_result.column->isNullable()) + { + auto result_column = IColumn::mutate(std::move(lambda_result.column)); + auto * column_nullable = assert_cast(result_column.get()); + auto & null_map = column_nullable->getNullMapData(); + auto nested_column = IColumn::mutate(std::move(column_nullable->getNestedColumnPtr())); + auto & nested_data = assert_cast(nested_column.get())->getData(); + for (size_t i = 0; i != nested_data.size(); ++i) + { + if (null_map[i]) + nested_data[i] = 0; + } + lambda_result.column = std::move(nested_column); + } + } + + return Impl::execute(*column_first_array, lambda_result.column); } } }; diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 72380c0a2bf..b0a7daac522 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -20,6 +20,8 @@ public: } bool useDefaultImplementationForNulls() const override { return false; } + /// array(..., Nothing, ...) -> Array(..., Nothing, ...) + bool useDefaultImplementationForNothing() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } bool isVariadic() const override { return true; } diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index a533cb2c0cc..2ef1cab4647 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,8 +7,7 @@ #include #include #include - -#include +#include "base/range.h" namespace DB { @@ -18,40 +18,98 @@ namespace ErrorCodes extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; } -template -struct LpDistance +struct L1Distance { - static inline String name = "L" + std::to_string(N); - template - static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, PaddedPODArray & array) + static inline String name = "L1"; + + template + struct State { - auto norms = (left - right).colwise().template lpNorm(); - array.reserve(norms.size()); - // array.insert() failed to work with Eigen iterators - for (auto n : norms) - array.push_back(n); + FloatType sum = 0; + }; + + template + static void accumulate(State & state, FirstArgType x, SecondArgType y) + { + state.sum += fabs(x - y); + } + + template + static ResultType finalize(const State & state) + { + return state.sum; } }; -struct LinfDistance : LpDistance +struct L2Distance { - static inline String name = "Linf"; + static inline String name = "L2"; + + template + struct State + { + FloatType sum = 0; + }; + + template + static void accumulate(State & state, FirstArgType x, SecondArgType y) + { + state.sum += (x - y) * (x - y); + } + + template + static ResultType finalize(const State & state) + { + return sqrt(state.sum); + } }; +struct LinfDistance +{ + static inline String name = "Linf"; + + template + struct State + { + FloatType dist = 0; + }; + + template + static void accumulate(State & state, FirstArgType x, SecondArgType y) + { + state.dist = fmax(state.dist, fabs(x - y)); + } + + template + static ResultType finalize(const State & state) + { + return state.dist; + } +}; struct CosineDistance { static inline String name = "Cosine"; - template - static void compute(const Eigen::MatrixX & left, const Eigen::MatrixX & right, PaddedPODArray & array) + + template + struct State { - auto prod = left.cwiseProduct(right).colwise().sum(); - auto nx = left.colwise().norm(); - auto ny = right.colwise().norm(); - auto nm = nx.cwiseProduct(ny).cwiseInverse(); - auto dist = 1.0 - prod.cwiseProduct(nm).array(); - array.reserve(dist.size()); - for (auto d : dist) - array.push_back(d); + FloatType dot_prod = 0; + FloatType x_squared = 0; + FloatType y_squared = 0; + }; + + template + static void accumulate(State & state, FirstArgType x, SecondArgType y) + { + state.dot_prod += x * y; + state.x_squared += x * x; + state.y_squared += y * y; + } + + template + static ResultType finalize(const State & state) + { + return 1 - state.dot_prod / sqrt(state.x_squared * state.y_squared); } }; @@ -102,144 +160,197 @@ public: } ColumnPtr - executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override + executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - DataTypePtr type_x = typeid_cast(arguments[0].type.get())->getNestedType(); - DataTypePtr type_y = typeid_cast(arguments[1].type.get())->getNestedType(); - - ColumnPtr col_x = arguments[0].column->convertToFullColumnIfConst(); - ColumnPtr col_y = arguments[1].column->convertToFullColumnIfConst(); - - const auto * arr_x = assert_cast(col_x.get()); - const auto * arr_y = assert_cast(col_y.get()); - - auto result = result_type->createColumn(); switch (result_type->getTypeId()) { case TypeIndex::Float32: - executeWithType(*arr_x, *arr_y, type_x, type_y, result); + return executeWithResultType(arguments, input_rows_count); break; case TypeIndex::Float64: - executeWithType(*arr_x, *arr_y, type_x, type_y, result); + return executeWithResultType(arguments, input_rows_count); break; default: throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected result type."); } - return result; } + +#define SUPPORTED_TYPES(action) \ + action(UInt8) \ + action(UInt16) \ + action(UInt32) \ + action(UInt64) \ + action(Int8) \ + action(Int16) \ + action(Int32) \ + action(Int64) \ + action(Float32) \ + action(Float64) + + private: - template - void executeWithType( - const ColumnArray & array_x, - const ColumnArray & array_y, - const DataTypePtr & type_x, - const DataTypePtr & type_y, - MutableColumnPtr & column) const + template + ColumnPtr executeWithResultType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { - Eigen::MatrixX mx, my; - columnToMatrix(array_x, type_x, mx); - columnToMatrix(array_y, type_y, my); + DataTypePtr type_x = typeid_cast(arguments[0].type.get())->getNestedType(); - if (mx.rows() && my.rows() && mx.rows() != my.rows()) + /// Dynamic disaptch based on the 1st argument type + switch (type_x->getTypeId()) { - throw Exception( - ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, - "Arguments of function {} have different array sizes: {} and {}", - getName(), mx.rows(), my.rows()); - } - auto & data = assert_cast &>(*column).getData(); - Kernel::compute(mx, my, data); - } + #define ON_TYPE(type) \ + case TypeIndex::type: \ + return executeWithFirstType(arguments, input_rows_count); \ + break; - template - void columnToMatrix(const ColumnArray & array, const DataTypePtr & nested_type, Eigen::MatrixX & mat) const - { - const auto & offsets = array.getOffsets(); - size_t cols = offsets.size(); - size_t rows = cols > 0 ? offsets.front() : 0; + SUPPORTED_TYPES(ON_TYPE) + #undef ON_TYPE - ColumnArray::Offset prev = 0; - for (ColumnArray::Offset off : offsets) - { - if (off - prev != rows) - throw Exception( - ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, - "Arrays in a column passed to function {} have different sizes: {} and {}", - getName(), rows, off - prev); - prev = off; - } - - switch (nested_type->getTypeId()) - { - case TypeIndex::UInt8: - fillMatrix(mat, array, rows, cols); - break; - case TypeIndex::UInt16: - fillMatrix(mat, array, rows, cols); - break; - case TypeIndex::UInt32: - fillMatrix(mat, array, rows, cols); - break; - case TypeIndex::UInt64: - fillMatrix(mat, array, rows, cols); - break; - case TypeIndex::Int8: - fillMatrix(mat, array, rows, cols); - break; - case TypeIndex::Int16: - fillMatrix(mat, array, rows, cols); - break; - case TypeIndex::Int32: - fillMatrix(mat, array, rows, cols); - break; - case TypeIndex::Int64: - fillMatrix(mat, array, rows, cols); - break; - case TypeIndex::Float32: - fillMatrix(mat, array, rows, cols); - break; - case TypeIndex::Float64: - fillMatrix(mat, array, rows, cols); - break; default: throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", - getName(), nested_type->getName()); + getName(), type_x->getName()); } } - // optimize for float/ double - template - requires std::is_same_v - void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array, size_t rows, size_t cols) const + template + ColumnPtr executeWithFirstType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { - const auto & data = typeid_cast &>(array.getData()).getData(); - mat = Eigen::Map>(data.data(), rows, cols); + DataTypePtr type_y = typeid_cast(arguments[1].type.get())->getNestedType(); + + /// Dynamic disaptch based on the 2nd argument type + switch (type_y->getTypeId()) + { + #define ON_TYPE(type) \ + case TypeIndex::type: \ + return executeWithTypes(arguments[0].column, arguments[1].column, input_rows_count); \ + break; + + SUPPORTED_TYPES(ON_TYPE) + #undef ON_TYPE + + default: + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Arguments of function {} has nested type {}. " + "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", + getName(), type_y->getName()); + } } - template - void fillMatrix(Eigen::MatrixX & mat, const ColumnArray & array, size_t rows, size_t cols) const + template + ColumnPtr executeWithTypes(ColumnPtr col_x, ColumnPtr col_y, size_t input_rows_count) const { - const auto & data = typeid_cast &>(array.getData()).getData(); - mat.resize(rows, cols); - for (size_t col = 0; col < cols; ++col) + if (typeid_cast(col_x.get())) { - for (size_t row = 0; row < rows; ++row) + return executeWithTypesFirstArgConst(col_x, col_y, input_rows_count); + } + else if (typeid_cast(col_y.get())) + { + return executeWithTypesFirstArgConst(col_y, col_x, input_rows_count); + } + + col_x = col_x->convertToFullColumnIfConst(); + col_y = col_y->convertToFullColumnIfConst(); + + const auto & array_x = *assert_cast(col_x.get()); + const auto & array_y = *assert_cast(col_y.get()); + + const auto & data_x = typeid_cast &>(array_x.getData()).getData(); + const auto & data_y = typeid_cast &>(array_y.getData()).getData(); + + const auto & offsets_x = array_x.getOffsets(); + const auto & offsets_y = array_y.getOffsets(); + + /// Check that arrays in both columns are the sames size + for (size_t row = 0; row < offsets_x.size(); ++row) + { + if (unlikely(offsets_x[row] != offsets_y[row])) { - size_t off = col * rows; - mat(row, col) = static_cast(data[off + row]); + ColumnArray::Offset prev_offset = row > 0 ? offsets_x[row] : 0; + throw Exception( + ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + "Arguments of function {} have different array sizes: {} and {}", + getName(), offsets_x[row] - prev_offset, offsets_y[row] - prev_offset); } } + + auto result = ColumnVector::create(input_rows_count); + auto & result_data = result->getData(); + + /// Do the actual computation + ColumnArray::Offset prev = 0; + size_t row = 0; + for (auto off : offsets_x) + { + typename Kernel::template State state; + for (; prev < off; ++prev) + { + Kernel::accumulate(state, data_x[prev], data_y[prev]); + } + result_data[row] = Kernel::finalize(state); + row++; + } + return result; } + + /// Special case when the 1st parameter is Const + template + ColumnPtr executeWithTypesFirstArgConst(ColumnPtr col_x, ColumnPtr col_y, size_t input_rows_count) const + { + col_x = assert_cast(col_x.get())->getDataColumnPtr(); + col_y = col_y->convertToFullColumnIfConst(); + + const auto & array_x = *assert_cast(col_x.get()); + const auto & array_y = *assert_cast(col_y.get()); + + const auto & data_x = typeid_cast &>(array_x.getData()).getData(); + const auto & data_y = typeid_cast &>(array_y.getData()).getData(); + + const auto & offsets_x = array_x.getOffsets(); + const auto & offsets_y = array_y.getOffsets(); + + /// Check that arrays in both columns are the sames size + ColumnArray::Offset prev_offset = 0; + for (size_t row : collections::range(0, offsets_y.size())) + { + if (unlikely(offsets_x[0] != offsets_y[row] - prev_offset)) + { + throw Exception( + ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, + "Arguments of function {} have different array sizes: {} and {}", + getName(), offsets_x[0], offsets_y[row] - prev_offset); + } + prev_offset = offsets_y[row]; + } + + auto result = ColumnVector::create(input_rows_count); + auto & result_data = result->getData(); + + /// Do the actual computation + ColumnArray::Offset prev = 0; + size_t row = 0; + for (auto off : offsets_y) + { + typename Kernel::template State state; + for (size_t i = 0; prev < off; ++i, ++prev) + { + Kernel::accumulate(state, data_x[i], data_y[prev]); + } + result_data[row] = Kernel::finalize(state); + row++; + } + return result; + } + }; void registerFunctionArrayDistance(FunctionFactory & factory) { - factory.registerFunction>>(); - factory.registerFunction>>(); + factory.registerFunction>(); + factory.registerFunction>(); factory.registerFunction>(); factory.registerFunction>(); } diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index 20fe85d7491..587c65a49ca 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -7,8 +9,6 @@ #include #include -#include - namespace DB { namespace ErrorCodes @@ -17,26 +17,59 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -template -struct LpNorm +struct L1Norm { - static inline String name = "L" + std::to_string(N); - template - static void compute(const std::vector> & vec, PaddedPODArray & array) + static inline String name = "L1"; + + template + inline static ResultType accumulate(ResultType result, ArgumentType value) { - array.reserve(vec.size()); - for (const auto & v : vec) - { - array.push_back(v.template lpNorm()); - } + return result + fabs(value); + } + + template + inline static ResultType finalize(ResultType result) + { + return result; } }; -struct LinfNorm : LpNorm +struct L2Norm +{ + static inline String name = "L2"; + + template + inline static ResultType accumulate(ResultType result, ArgumentType value) + { + return result + value * value; + } + + template + inline static ResultType finalize(ResultType result) + { + return sqrt(result); + } +}; + + +struct LinfNorm { static inline String name = "Linf"; + + template + inline static ResultType accumulate(ResultType result, ArgumentType value) + { + return fmax(result, fabs(value)); + } + + template + inline static ResultType finalize(ResultType result) + { + return result; + } }; + template class FunctionArrayNorm : public IFunction { @@ -84,72 +117,53 @@ public: } ColumnPtr - executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override + executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { DataTypePtr type = typeid_cast(arguments[0].type.get())->getNestedType(); ColumnPtr column = arguments[0].column->convertToFullColumnIfConst(); const auto * arr = assert_cast(column.get()); - auto result = result_type->createColumn(); switch (result_type->getTypeId()) { case TypeIndex::Float32: - executeWithType(*arr, type, result); + return executeWithResultType(*arr, type, input_rows_count); break; case TypeIndex::Float64: - executeWithType(*arr, type, result); + return executeWithResultType(*arr, type, input_rows_count); break; default: throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected result type."); } - return result; } private: - template - void executeWithType(const ColumnArray & array, const DataTypePtr & type, MutableColumnPtr & column) const - { - std::vector> vec; - columnToVectors(array, type, vec); - auto & data = assert_cast &>(*column).getData(); - Kernel::compute(vec, data); - } - template - void columnToVectors(const ColumnArray & array, const DataTypePtr & nested_type, std::vector> & vec) const +#define SUPPORTED_TYPES(action) \ + action(UInt8) \ + action(UInt16) \ + action(UInt32) \ + action(UInt64) \ + action(Int8) \ + action(Int16) \ + action(Int32) \ + action(Int64) \ + action(Float32) \ + action(Float64) + + + template + ColumnPtr executeWithResultType(const ColumnArray & array, const DataTypePtr & nested_type, size_t input_rows_count) const { switch (nested_type->getTypeId()) { - case TypeIndex::UInt8: - fillVectors(vec, array); - break; - case TypeIndex::UInt16: - fillVectors(vec, array); - break; - case TypeIndex::UInt32: - fillVectors(vec, array); - break; - case TypeIndex::UInt64: - fillVectors(vec, array); - break; - case TypeIndex::Int8: - fillVectors(vec, array); - break; - case TypeIndex::Int16: - fillVectors(vec, array); - break; - case TypeIndex::Int32: - fillVectors(vec, array); - break; - case TypeIndex::Int64: - fillVectors(vec, array); - break; - case TypeIndex::Float32: - fillVectors(vec, array); - break; - case TypeIndex::Float64: - fillVectors(vec, array); + #define ON_TYPE(type) \ + case TypeIndex::type: \ + return executeWithTypes(array, input_rows_count); \ break; + + SUPPORTED_TYPES(ON_TYPE) + #undef ON_TYPE + default: throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, @@ -159,46 +173,35 @@ private: } } - template - requires std::is_same_v - void fillVectors(std::vector> & vec, const ColumnArray & array) const + template + static ColumnPtr executeWithTypes(const ColumnArray & array, size_t input_rows_count) { - const auto & data = typeid_cast &>(array.getData()).getData(); + const auto & data = typeid_cast &>(array.getData()).getData(); const auto & offsets = array.getOffsets(); - vec.reserve(offsets.size()); - ColumnArray::Offset prev = 0; - for (auto off : offsets) - { - vec.emplace_back(Eigen::Map>(data.data() + prev, off - prev)); - prev = off; - } - } - template - void fillVectors(std::vector> & vec, const ColumnArray & array) const - { - const auto & data = typeid_cast &>(array.getData()).getData(); - const auto & offsets = array.getOffsets(); - vec.reserve(offsets.size()); + auto result_col = ColumnVector::create(input_rows_count); + auto & result_data = result_col->getData(); ColumnArray::Offset prev = 0; + size_t row = 0; for (auto off : offsets) { - Eigen::VectorX mat(off - prev); - for (ColumnArray::Offset row = 0; row + prev < off; ++row) + Float64 result = 0; + for (; prev < off; ++prev) { - mat[row] = static_cast(data[prev + row]); + result = Kernel::accumulate(result, data[prev]); } - prev = off; - vec.emplace_back(mat); + result_data[row] = Kernel::finalize(result); + row++; } + return result_col; } }; void registerFunctionArrayNorm(FunctionFactory & factory) { - factory.registerFunction>>(); - factory.registerFunction>>(); + factory.registerFunction>(); + factory.registerFunction>(); factory.registerFunction>(); } diff --git a/src/Functions/assumeNotNull.cpp b/src/Functions/assumeNotNull.cpp index b180931a83a..8f999af9ef0 100644 --- a/src/Functions/assumeNotNull.cpp +++ b/src/Functions/assumeNotNull.cpp @@ -7,6 +7,12 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + namespace { @@ -45,6 +51,9 @@ public: { const ColumnPtr & col = arguments[0].column; + if (arguments[0].type->onlyNull() && !col->empty()) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot create non-empty column with type Nothing"); + if (const auto * nullable_col = checkAndGetColumn(*col)) return nullable_col->getNestedColumnPtr(); else diff --git a/src/Functions/castOrDefault.cpp b/src/Functions/castOrDefault.cpp index f7b93ec2e83..2e9f13545fd 100644 --- a/src/Functions/castOrDefault.cpp +++ b/src/Functions/castOrDefault.cpp @@ -52,6 +52,7 @@ public: ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForNothing() const override { return false; } bool useDefaultImplementationForConstants() const override { return false; } bool useDefaultImplementationForLowCardinalityColumns() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } @@ -194,6 +195,7 @@ private: bool isVariadic() const override { return true; } bool useDefaultImplementationForNulls() const override { return impl.useDefaultImplementationForNulls(); } + bool useDefaultImplementationForNothing() const override { return impl.useDefaultImplementationForNothing(); } bool useDefaultImplementationForLowCardinalityColumns() const override { return impl.useDefaultImplementationForLowCardinalityColumns();} bool useDefaultImplementationForConstants() const override { return impl.useDefaultImplementationForConstants();} bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & arguments) const override diff --git a/src/Functions/ilike.cpp b/src/Functions/ilike.cpp index 116c945e04f..1222cc48d07 100644 --- a/src/Functions/ilike.cpp +++ b/src/Functions/ilike.cpp @@ -12,7 +12,7 @@ struct NameILike static constexpr auto name = "ilike"; }; -using ILikeImpl = MatchImpl; +using ILikeImpl = MatchImpl; using FunctionILike = FunctionsStringSearch; } diff --git a/src/Functions/isConstant.cpp b/src/Functions/isConstant.cpp index 7ea3e26cb82..6d76cfc1dcc 100644 --- a/src/Functions/isConstant.cpp +++ b/src/Functions/isConstant.cpp @@ -27,6 +27,8 @@ public: bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForNothing() const override { return false; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } size_t getNumberOfArguments() const override diff --git a/src/Functions/like.h b/src/Functions/like.h index a00891ec64c..edb738d393b 100644 --- a/src/Functions/like.h +++ b/src/Functions/like.h @@ -11,7 +11,7 @@ struct NameLike static constexpr auto name = "like"; }; -using LikeImpl = MatchImpl; +using LikeImpl = MatchImpl; using FunctionLike = FunctionsStringSearch; } diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 471d6fc575c..8c891fdec81 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -65,6 +65,8 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } bool useDefaultImplementationForNulls() const override { return false; } + /// map(..., Nothing) -> Map(..., Nothing) + bool useDefaultImplementationForNothing() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override diff --git a/src/Functions/match.cpp b/src/Functions/match.cpp index 69dc1a3d99a..4c329701464 100644 --- a/src/Functions/match.cpp +++ b/src/Functions/match.cpp @@ -13,7 +13,7 @@ struct NameMatch static constexpr auto name = "match"; }; -using FunctionMatch = FunctionsStringSearch>; +using FunctionMatch = FunctionsStringSearch>; } diff --git a/src/Functions/normalizeString.cpp b/src/Functions/normalizeString.cpp index 8989e56d0d3..9b1d1292d2c 100644 --- a/src/Functions/normalizeString.cpp +++ b/src/Functions/normalizeString.cpp @@ -106,7 +106,7 @@ struct NormalizeUTF8Impl size_t from_size = offsets[i] - current_from_offset - 1; from_uchars.resize(from_size + 1); - int32_t from_code_points; + int32_t from_code_points = 0; u_strFromUTF8( from_uchars.data(), from_uchars.size(), @@ -133,7 +133,7 @@ struct NormalizeUTF8Impl if (res_data.size() < max_to_size) res_data.resize(max_to_size); - int32_t to_size; + int32_t to_size = 0; u_strToUTF8( reinterpret_cast(&res_data[current_to_offset]), res_data.size() - current_to_offset, @@ -151,6 +151,8 @@ struct NormalizeUTF8Impl current_from_offset = offsets[i]; } + + res_data.resize(current_to_offset); } [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &) diff --git a/src/Functions/notILike.cpp b/src/Functions/notILike.cpp index be40e2b989e..b5e06ac55f4 100644 --- a/src/Functions/notILike.cpp +++ b/src/Functions/notILike.cpp @@ -12,7 +12,7 @@ struct NameNotILike static constexpr auto name = "notILike"; }; -using NotILikeImpl = MatchImpl; +using NotILikeImpl = MatchImpl; using FunctionNotILike = FunctionsStringSearch; } diff --git a/src/Functions/notLike.cpp b/src/Functions/notLike.cpp index 7c4ea8ab2dc..7fa1b6f9122 100644 --- a/src/Functions/notLike.cpp +++ b/src/Functions/notLike.cpp @@ -12,7 +12,7 @@ struct NameNotLike static constexpr auto name = "notLike"; }; -using FunctionNotLike = FunctionsStringSearch>; +using FunctionNotLike = FunctionsStringSearch>; } diff --git a/src/Functions/toColumnTypeName.cpp b/src/Functions/toColumnTypeName.cpp index 95c35243567..345de2954c4 100644 --- a/src/Functions/toColumnTypeName.cpp +++ b/src/Functions/toColumnTypeName.cpp @@ -26,6 +26,8 @@ public: bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForNothing() const override { return false; } + bool isShortCircuit(ShortCircuitSettings & settings, size_t /*number_of_arguments*/) const override { settings.enable_lazy_execution_for_first_argument = true; diff --git a/src/Functions/toNullable.cpp b/src/Functions/toNullable.cpp index b7fe831f4ff..16d9f9198cd 100644 --- a/src/Functions/toNullable.cpp +++ b/src/Functions/toNullable.cpp @@ -28,6 +28,7 @@ public: size_t getNumberOfArguments() const override { return 1; } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForNothing() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } diff --git a/src/Functions/toTypeName.cpp b/src/Functions/toTypeName.cpp index 41567ac1ee3..f3af49315ed 100644 --- a/src/Functions/toTypeName.cpp +++ b/src/Functions/toTypeName.cpp @@ -30,6 +30,8 @@ public: bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForNothing() const override { return false; } + bool isShortCircuit(ShortCircuitSettings & settings, size_t /*number_of_arguments*/) const override { settings.enable_lazy_execution_for_first_argument = false; diff --git a/src/Functions/tuple.cpp b/src/Functions/tuple.cpp index 6d5c53c0770..4238b12157a 100644 --- a/src/Functions/tuple.cpp +++ b/src/Functions/tuple.cpp @@ -52,6 +52,8 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } bool useDefaultImplementationForNulls() const override { return false; } + /// tuple(..., Nothing, ...) -> Tuple(..., Nothing, ...) + bool useDefaultImplementationForNothing() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 846b4762cb3..21e943b36ef 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -256,6 +256,7 @@ void readString(String & s, ReadBuffer & buf) template void readStringInto>(PaddedPODArray & s, ReadBuffer & buf); template void readStringInto(String & s, ReadBuffer & buf); +template void readStringInto(NullOutput & s, ReadBuffer & buf); template void readStringUntilEOFInto(Vector & s, ReadBuffer & buf) @@ -617,6 +618,12 @@ void readBackQuotedStringWithSQLStyle(String & s, ReadBuffer & buf) readBackQuotedStringInto(s, buf); } +template +concept WithResize = requires (T value) +{ + { value.resize(1) }; + { value.size() } -> std::integral<>; +}; template void readCSVStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::CSV & settings) @@ -700,16 +707,18 @@ void readCSVStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::CSV & if (!buf.hasPendingData()) continue; - /** CSV format can contain insignificant spaces and tabs. - * Usually the task of skipping them is for the calling code. - * But in this case, it will be difficult to do this, so remove the trailing whitespace by ourself. - */ - size_t size = s.size(); - while (size > 0 - && (s[size - 1] == ' ' || s[size - 1] == '\t')) - --size; + if constexpr (WithResize) + { + /** CSV format can contain insignificant spaces and tabs. + * Usually the task of skipping them is for the calling code. + * But in this case, it will be difficult to do this, so remove the trailing whitespace by ourself. + */ + size_t size = s.size(); + while (size > 0 && (s[size - 1] == ' ' || s[size - 1] == '\t')) + --size; - s.resize(size); + s.resize(size); + } return; } } @@ -741,6 +750,7 @@ void readCSVField(String & s, ReadBuffer & buf, const FormatSettings::CSV & sett } template void readCSVStringInto>(PaddedPODArray & s, ReadBuffer & buf, const FormatSettings::CSV & settings); +template void readCSVStringInto(NullOutput & s, ReadBuffer & buf, const FormatSettings::CSV & settings); template @@ -1313,8 +1323,8 @@ void skipToNextRowOrEof(PeekableReadBuffer & buf, const String & row_after_delim } // Use PeekableReadBuffer to copy field to string after parsing. -template -static void readParsedValueIntoString(String & s, ReadBuffer & buf, ParseFunc parse_func) +template +static void readParsedValueInto(Vector & s, ReadBuffer & buf, ParseFunc parse_func) { PeekableReadBuffer peekable_buf(buf); peekable_buf.setCheckpoint(); @@ -1326,8 +1336,8 @@ static void readParsedValueIntoString(String & s, ReadBuffer & buf, ParseFunc pa peekable_buf.position() = end; } -template -static void readQuotedFieldInBrackets(String & s, ReadBuffer & buf) +template +static void readQuotedFieldInBracketsInto(Vector & s, ReadBuffer & buf) { assertChar(opening_bracket, buf); s.push_back(opening_bracket); @@ -1363,10 +1373,9 @@ static void readQuotedFieldInBrackets(String & s, ReadBuffer & buf) } } -void readQuotedField(String & s, ReadBuffer & buf) +template +void readQuotedFieldInto(Vector & s, ReadBuffer & buf) { - s.clear(); - if (buf.eof()) return; @@ -1386,11 +1395,11 @@ void readQuotedField(String & s, ReadBuffer & buf) s.push_back('\''); } else if (*buf.position() == '[') - readQuotedFieldInBrackets<'[', ']'>(s, buf); + readQuotedFieldInBracketsInto<'[', ']'>(s, buf); else if (*buf.position() == '(') - readQuotedFieldInBrackets<'(', ')'>(s, buf); + readQuotedFieldInBracketsInto<'(', ')'>(s, buf); else if (*buf.position() == '{') - readQuotedFieldInBrackets<'{', '}'>(s, buf); + readQuotedFieldInBracketsInto<'{', '}'>(s, buf); else if (checkCharCaseInsensitive('n', buf)) { /// NULL or NaN @@ -1423,15 +1432,23 @@ void readQuotedField(String & s, ReadBuffer & buf) Float64 tmp; readFloatText(tmp, in); }; - readParsedValueIntoString(s, buf, parse_func); + readParsedValueInto(s, buf, parse_func); } } +template void readQuotedFieldInto(NullOutput & s, ReadBuffer & buf); + +void readQuotedField(String & s, ReadBuffer & buf) +{ + s.clear(); + readQuotedFieldInto(s, buf); +} + void readJSONField(String & s, ReadBuffer & buf) { s.clear(); auto parse_func = [](ReadBuffer & in) { skipJSONField(in, "json_field"); }; - readParsedValueIntoString(s, buf, parse_func); + readParsedValueInto(s, buf, parse_func); } } diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 496b8000441..e11c57947d1 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -618,6 +618,8 @@ void readStringUntilNewlineInto(Vector & s, ReadBuffer & buf); struct NullOutput { void append(const char *, size_t) {} + void append(const char *) {} + void append(const char *, const char *) {} void push_back(char) {} /// NOLINT }; @@ -931,12 +933,29 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re ++buf.position(); /// Keep sign of fractional part the same with whole part if datetime64 is negative - /// 1965-12-12 12:12:12.123 => whole = -127914468, fraction = 123(sign>0) -> new whole = -127914467, new fraction = 877(sign<0) + /// Case1: + /// 1965-12-12 12:12:12.123 + /// => whole = -127914468, fractional = 123(coefficient>0) + /// => new whole = -127914467, new fractional = 877(coefficient<0) + /// + /// Case2: + /// 1969-12-31 23:59:59.123 + /// => whole = -1, fractional = 123(coefficient>0) + /// => new whole = 0, new fractional = -877(coefficient>0) if (components.whole < 0 && components.fractional != 0) { const auto scale_multiplier = DecimalUtils::scaleMultiplier(scale); ++components.whole; - components.fractional = scale_multiplier - components.fractional; + if (components.whole) + { + /// whole keep the sign, fractional should be non-negative + components.fractional = scale_multiplier - components.fractional; + } + else + { + /// when whole is zero, fractional should keep the sign + components.fractional = components.fractional - scale_multiplier; + } } } /// 9908870400 is time_t value for 2184-01-01 UTC (a bit over the last year supported by DateTime64) @@ -1425,6 +1444,9 @@ struct PcgDeserializer } }; +template +void readQuotedFieldInto(Vector & s, ReadBuffer & buf); + void readQuotedField(String & s, ReadBuffer & buf); void readJSONField(String & s, ReadBuffer & buf); diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 8547a0af1cd..5eab75f14b1 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -805,11 +805,21 @@ inline void writeDateTimeText(DateTime64 datetime64, UInt32 scale, WriteBuffer & scale = scale > MaxScale ? MaxScale : scale; auto components = DecimalUtils::split(datetime64, scale); - /// -127914467.877 => whole = -127914467, fraction = 877 => new whole = -127914468(1965-12-12 12:12:12), new fraction = 123(.123) => 1965-12-12 12:12:12.123 - if (components.whole < 0 && components.fractional != 0) + /// Case1: + /// -127914467.877 + /// => whole = -127914467, fraction = 877(After DecimalUtils::split) + /// => new whole = -127914468(1965-12-12 12:12:12), new fraction = 1000 - 877 = 123(.123) + /// => 1965-12-12 12:12:12.123 + /// + /// Case2: + /// -0.877 + /// => whole = 0, fractional = -877(After DecimalUtils::split) + /// => whole = -1(1969-12-31 23:59:59), fractional = 1000 + (-877) = 123(.123) + using T = typename DateTime64::NativeType; + if (datetime64.value < 0 && components.fractional) { + components.fractional = DecimalUtils::scaleMultiplier(scale) + (components.whole ? T(-1) : T(1)) * components.fractional; --components.whole; - components.fractional = DecimalUtils::scaleMultiplier(scale) - components.fractional; } writeDateTimeText(LocalDateTime(components.whole, time_zone), buf); @@ -989,7 +999,12 @@ void writeText(Decimal x, UInt32 scale, WriteBuffer & ostr, bool trailing_zer { part = DecimalUtils::getFractionalPart(x, scale); if (part || trailing_zeros) + { + if (part < 0) + part *= T(-1); + writeDecimalFractional(part, scale, ostr, trailing_zeros); + } } } diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index f796a55ff72..2fc9b51674f 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -997,8 +997,8 @@ void ActionsDAG::addMaterializingOutputActions() const ActionsDAG::Node & ActionsDAG::materializeNode(const Node & node) { - FunctionOverloadResolverPtr func_builder_materialize = std::make_unique( - std::make_shared()); + FunctionOverloadResolverPtr func_builder_materialize + = std::make_unique(std::make_shared()); const auto & name = node.result_name; const auto * func = &addFunction(func_builder_materialize, {&node}, {}); @@ -1102,7 +1102,8 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( const auto * left_arg = dst_node; FunctionCastBase::Diagnostic diagnostic = {dst_node->result_name, res_elem.name}; - FunctionOverloadResolverPtr func_builder_cast = CastInternalOverloadResolver::createImpl(std::move(diagnostic)); + FunctionOverloadResolverPtr func_builder_cast + = CastInternalOverloadResolver::createImpl(std::move(diagnostic)); NodeRawConstPtrs children = { left_arg, right_arg }; dst_node = &actions_dag->addFunction(func_builder_cast, std::move(children), {}); @@ -1150,7 +1151,8 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( ActionsDAGPtr ActionsDAG::makeAddingColumnActions(ColumnWithTypeAndName column) { auto adding_column_action = std::make_shared(); - FunctionOverloadResolverPtr func_builder_materialize = std::make_unique(std::make_shared()); + FunctionOverloadResolverPtr func_builder_materialize + = std::make_unique(std::make_shared()); auto column_name = column.name; const auto * column_node = &adding_column_action->addColumn(std::move(column)); @@ -1612,7 +1614,7 @@ ConjunctionNodes getConjunctionNodes(ActionsDAG::Node * predicate, std::unordere std::stack stack; std::unordered_set visited_nodes; - stack.push(Frame{.node = predicate}); + stack.push({.node = predicate}); visited_nodes.insert(predicate); while (!stack.empty()) { @@ -1798,9 +1800,8 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( { Node * predicate = const_cast(tryFindInIndex(filter_name)); if (!predicate) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Index for ActionsDAG does not contain filter column name {}. DAG:\n{}", - filter_name, dumpDAG()); + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Index for ActionsDAG does not contain filter column name {}. DAG:\n{}", filter_name, dumpDAG()); /// If condition is constant let's do nothing. /// It means there is nothing to push down or optimization was already applied. @@ -1870,8 +1871,6 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( index_node = new_predicate; } } - - removeUnusedActions(false); } else { @@ -1926,10 +1925,9 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( predicate->function_base = predicate->function_builder->build(arguments); predicate->function = predicate->function_base->prepare(arguments); } - - removeUnusedActions(false); } + removeUnusedActions(false); return actions; } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 20aa6dbb6c7..1300bcc0475 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -342,8 +342,6 @@ struct ContextSharedPart /// Stop periodic reloading of the configuration files. /// This must be done first because otherwise the reloading may pass a changed config /// to some destroyed parts of ContextSharedPart. - if (access_control) - access_control->stopPeriodicReloadingUsersConfigs(); if (external_dictionaries_loader) external_dictionaries_loader->enablePeriodicUpdates(false); if (external_user_defined_executable_functions_loader) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 0e12c5e9e5a..2b2de84c314 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -287,7 +287,7 @@ void DDLWorker::scheduleTasks(bool reinitialized) Strings queue_nodes = zookeeper->getChildren(queue_dir, &queue_node_stat, queue_updated_event); size_t size_before_filtering = queue_nodes.size(); filterAndSortQueueNodes(queue_nodes); - /// The following message is too verbose, but it can be useful too debug mysterious test failures in CI + /// The following message is too verbose, but it can be useful to debug mysterious test failures in CI LOG_TRACE(log, "scheduleTasks: initialized={}, size_before_filtering={}, queue_size={}, " "entries={}..{}, " "first_failed_task_name={}, current_tasks_size={}, " diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index df53333b635..304cfa2f3f4 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -578,7 +578,12 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (col_decl.type) column.type = name_type_it->type; else + { column.type = defaults_sample_block.getByName(column.name).type; + /// set nullability for case of column declaration w/o type but with default expression + if ((col_decl.null_modifier && *col_decl.null_modifier) || make_columns_nullable) + column.type = makeNullable(column.type); + } column.default_desc.kind = columnDefaultKindFromString(col_decl.default_specifier); column.default_desc.expression = default_expr; diff --git a/src/Interpreters/InterpreterTransactionControlQuery.cpp b/src/Interpreters/InterpreterTransactionControlQuery.cpp index 61b2a4e865f..1e4868788ba 100644 --- a/src/Interpreters/InterpreterTransactionControlQuery.cpp +++ b/src/Interpreters/InterpreterTransactionControlQuery.cpp @@ -10,6 +10,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int INVALID_TRANSACTION; + extern const int UNKNOWN_STATUS_OF_TRANSACTION; } BlockIO InterpreterTransactionControlQuery::execute() @@ -55,7 +56,42 @@ BlockIO InterpreterTransactionControlQuery::executeCommit(ContextMutablePtr sess if (txn->getState() != MergeTreeTransaction::RUNNING) throw Exception(ErrorCodes::INVALID_TRANSACTION, "Transaction is not in RUNNING state"); - TransactionLog::instance().commitTransaction(txn); + TransactionsWaitCSNMode mode = query_context->getSettingsRef().wait_changes_become_visible_after_commit_mode; + CSN csn; + try + { + csn = TransactionLog::instance().commitTransaction(txn, /* throw_on_unknown_status */ mode != TransactionsWaitCSNMode::WAIT_UNKNOWN); + } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::UNKNOWN_STATUS_OF_TRANSACTION) + { + /// Detach transaction from current context if connection was lost and its status is unknown + session_context->setCurrentTransaction(NO_TRANSACTION_PTR); + } + throw; + } + + if (csn == Tx::CommittingCSN) + { + chassert(mode == TransactionsWaitCSNMode::WAIT_UNKNOWN); + + /// Try to wait for connection to be restored and its status to be loaded. + /// It's useful for testing. It allows to enable fault injection (after commit) without breaking tests. + txn->waitStateChange(Tx::CommittingCSN); + + if (txn->getState() == MergeTreeTransaction::ROLLED_BACK) + throw Exception(ErrorCodes::INVALID_TRANSACTION, "Transaction {} was rolled back", txn->tid); + if (txn->getState() != MergeTreeTransaction::COMMITTED) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Transaction {} has invalid state {}", txn->tid, txn->getState()); + + csn = txn->getCSN(); + } + + /// Wait for committed changes to become actually visible, so the next transaction in this session will see the changes + if (mode != TransactionsWaitCSNMode::ASYNC) + TransactionLog::instance().waitForCSNLoaded(csn); + session_context->setCurrentTransaction(NO_TRANSACTION_PTR); return {}; } @@ -67,6 +103,8 @@ BlockIO InterpreterTransactionControlQuery::executeRollback(ContextMutablePtr se throw Exception(ErrorCodes::INVALID_TRANSACTION, "There is no current transaction"); if (txn->getState() == MergeTreeTransaction::COMMITTED) throw Exception(ErrorCodes::LOGICAL_ERROR, "Transaction is in COMMITTED state"); + if (txn->getState() == MergeTreeTransaction::COMMITTING) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Transaction is in COMMITTING state"); if (txn->getState() == MergeTreeTransaction::RUNNING) TransactionLog::instance().rollbackTransaction(txn); diff --git a/src/Interpreters/InterpreterTransactionControlQuery.h b/src/Interpreters/InterpreterTransactionControlQuery.h index 05d3068e095..bf2dc7891a7 100644 --- a/src/Interpreters/InterpreterTransactionControlQuery.h +++ b/src/Interpreters/InterpreterTransactionControlQuery.h @@ -22,7 +22,7 @@ public: private: BlockIO executeBegin(ContextMutablePtr session_context); - static BlockIO executeCommit(ContextMutablePtr session_context); + BlockIO executeCommit(ContextMutablePtr session_context); static BlockIO executeRollback(ContextMutablePtr session_context); static BlockIO executeSetSnapshot(ContextMutablePtr session_context, UInt64 snapshot); diff --git a/src/Interpreters/JoinSwitcher.cpp b/src/Interpreters/JoinSwitcher.cpp index 480d105ebb6..34c8bb4cfd5 100644 --- a/src/Interpreters/JoinSwitcher.cpp +++ b/src/Interpreters/JoinSwitcher.cpp @@ -66,7 +66,7 @@ void JoinSwitcher::switchJoin() for (const auto & sample_column : right_sample_block) { positions.emplace_back(tmp_block.getPositionByName(sample_column.name)); - is_nullable.emplace_back(sample_column.type->isNullable()); + is_nullable.emplace_back(JoinCommon::isNullable(sample_column.type)); } } diff --git a/src/Interpreters/MergeTreeTransaction.cpp b/src/Interpreters/MergeTreeTransaction.cpp index c0d3cdfeb62..cab40f3c6db 100644 --- a/src/Interpreters/MergeTreeTransaction.cpp +++ b/src/Interpreters/MergeTreeTransaction.cpp @@ -38,13 +38,26 @@ void MergeTreeTransaction::setSnapshot(CSN new_snapshot) MergeTreeTransaction::State MergeTreeTransaction::getState() const { CSN c = csn.load(); - if (c == Tx::UnknownCSN || c == Tx::CommittingCSN) + if (c == Tx::UnknownCSN) return RUNNING; + if (c == Tx::CommittingCSN) + return COMMITTING; if (c == Tx::RolledBackCSN) return ROLLED_BACK; return COMMITTED; } +bool MergeTreeTransaction::waitStateChange(CSN current_state_csn) const +{ + CSN current_value = current_state_csn; + while (current_value == current_state_csn && !TransactionLog::instance().isShuttingDown()) + { + csn.wait(current_value); + current_value = csn.load(); + } + return current_value != current_state_csn; +} + void MergeTreeTransaction::checkIsNotCancelled() const { CSN c = csn.load(); @@ -158,7 +171,7 @@ void MergeTreeTransaction::addMutation(const StoragePtr & table, const String & bool MergeTreeTransaction::isReadOnly() const { std::lock_guard lock{mutex}; - assert((creating_parts.empty() && removing_parts.empty() && mutations.empty()) == storages.empty()); + chassert((creating_parts.empty() && removing_parts.empty() && mutations.empty()) == storages.empty()); return storages.empty(); } @@ -204,7 +217,7 @@ void MergeTreeTransaction::afterCommit(CSN assigned_csn) noexcept /// and we will be able to remove old entries from transaction log in ZK. /// It's not a problem if server crash before CSN is written, because we already have TID in data part and entry in the log. [[maybe_unused]] CSN prev_value = csn.exchange(assigned_csn); - assert(prev_value == Tx::CommittingCSN); + chassert(prev_value == Tx::CommittingCSN); for (const auto & part : creating_parts) { part->version.creation_csn.store(csn); @@ -321,7 +334,7 @@ String MergeTreeTransaction::dumpDescription() const { String info = fmt::format("{} (created by {}, {})", part->name, part->version.getCreationTID(), part->version.creation_csn); std::get<1>(storage_to_changes[&(part->storage)]).push_back(std::move(info)); - assert(!part->version.creation_csn || part->version.creation_csn <= snapshot); + chassert(!part->version.creation_csn || part->version.creation_csn <= snapshot); } for (const auto & mutation : mutations) diff --git a/src/Interpreters/MergeTreeTransaction.h b/src/Interpreters/MergeTreeTransaction.h index 7ebea450dd0..309b8e3eeff 100644 --- a/src/Interpreters/MergeTreeTransaction.h +++ b/src/Interpreters/MergeTreeTransaction.h @@ -26,6 +26,7 @@ public: enum State { RUNNING, + COMMITTING, COMMITTED, ROLLED_BACK, }; @@ -55,6 +56,11 @@ public: Float64 elapsedSeconds() const { return elapsed.elapsedSeconds(); } + /// Waits for transaction state to become not equal to the state corresponding to current_state_csn + bool waitStateChange(CSN current_state_csn) const; + + CSN getCSN() const { return csn; } + private: scope_guard beforeCommit(); void afterCommit(CSN assigned_csn) noexcept; diff --git a/src/Interpreters/MergeTreeTransactionHolder.cpp b/src/Interpreters/MergeTreeTransactionHolder.cpp index bf63a471282..2944fb78b76 100644 --- a/src/Interpreters/MergeTreeTransactionHolder.cpp +++ b/src/Interpreters/MergeTreeTransactionHolder.cpp @@ -53,7 +53,7 @@ void MergeTreeTransactionHolder::onDestroy() noexcept { try { - TransactionLog::instance().commitTransaction(txn); + TransactionLog::instance().commitTransaction(txn, /* throw_on_unknown_status */ false); return; } catch (...) diff --git a/src/Interpreters/TransactionLog.cpp b/src/Interpreters/TransactionLog.cpp index e65630d907b..4f0e79297b8 100644 --- a/src/Interpreters/TransactionLog.cpp +++ b/src/Interpreters/TransactionLog.cpp @@ -21,6 +21,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int UNKNOWN_STATUS_OF_TRANSACTION; } static void tryWriteEventToSystemLog(Poco::Logger * log, ContextPtr context, @@ -52,6 +53,8 @@ TransactionLog::TransactionLog() zookeeper_path = global_context->getConfigRef().getString("transaction_log.zookeeper_path", "/clickhouse/txn"); zookeeper_path_log = zookeeper_path + "/log"; + fault_probability_before_commit = global_context->getConfigRef().getDouble("transaction_log.fault_probability_before_commit", 0); + fault_probability_after_commit = global_context->getConfigRef().getDouble("transaction_log.fault_probability_after_commit", 0); loadLogFromZooKeeper(); @@ -145,24 +148,29 @@ void TransactionLog::loadEntries(Strings::const_iterator beg, Strings::const_ite NOEXCEPT_SCOPE; LockMemoryExceptionInThread lock_memory_tracker(VariableContext::Global); - std::lock_guard lock{mutex}; - for (const auto & entry : loaded) { - if (entry.first == Tx::EmptyTID.getHash()) - continue; + std::lock_guard lock{mutex}; + for (const auto & entry : loaded) + { + if (entry.first == Tx::EmptyTID.getHash()) + continue; - tid_to_csn.emplace(entry.first, entry.second); + tid_to_csn.emplace(entry.first, entry.second); + } + last_loaded_entry = last_entry; + } + { + std::lock_guard lock{running_list_mutex}; + latest_snapshot = loaded.back().second.csn; + local_tid_counter = Tx::MaxReservedLocalTID; } - last_loaded_entry = last_entry; - latest_snapshot = loaded.back().second.csn; - local_tid_counter = Tx::MaxReservedLocalTID; } void TransactionLog::loadLogFromZooKeeper() { - assert(!zookeeper); - assert(tid_to_csn.empty()); - assert(last_loaded_entry.empty()); + chassert(!zookeeper); + chassert(tid_to_csn.empty()); + chassert(last_loaded_entry.empty()); zookeeper = global_context->getZooKeeper(); /// We do not write local_tid_counter to disk or zk and maintain it only in memory. @@ -172,7 +180,7 @@ void TransactionLog::loadLogFromZooKeeper() if (code != Coordination::Error::ZOK) { /// Log probably does not exist, create it - assert(code == Coordination::Error::ZNONODE); + chassert(code == Coordination::Error::ZNONODE); zookeeper->createAncestors(zookeeper_path_log); Coordination::Requests ops; ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/tail_ptr", serializeCSN(Tx::MaxReservedCSN), zkutil::CreateMode::Persistent)); @@ -192,11 +200,11 @@ void TransactionLog::loadLogFromZooKeeper() /// 2. simplify log rotation /// 3. support 64-bit CSNs on top of Apache ZooKeeper (it uses Int32 for sequential numbers) Strings entries_list = zookeeper->getChildren(zookeeper_path_log, nullptr, log_updated_event); - assert(!entries_list.empty()); + chassert(!entries_list.empty()); std::sort(entries_list.begin(), entries_list.end()); loadEntries(entries_list.begin(), entries_list.end()); - assert(!last_loaded_entry.empty()); - assert(latest_snapshot == deserializeCSN(last_loaded_entry)); + chassert(!last_loaded_entry.empty()); + chassert(latest_snapshot == deserializeCSN(last_loaded_entry)); local_tid_counter = Tx::MaxReservedLocalTID; tail_ptr = deserializeCSN(zookeeper->get(zookeeper_path + "/tail_ptr")); @@ -208,19 +216,31 @@ void TransactionLog::runUpdatingThread() { try { - log_updated_event->wait(); + /// Do not wait if we have some transactions to finalize + if (!unknown_state_list_loaded.empty()) + log_updated_event->wait(); + if (stop_flag.load()) return; - if (getZooKeeper()->expired()) + bool connection_loss = getZooKeeper()->expired(); + if (connection_loss) { auto new_zookeeper = global_context->getZooKeeper(); - std::lock_guard lock{mutex}; - zookeeper = new_zookeeper; + { + std::lock_guard lock{mutex}; + zookeeper = new_zookeeper; + } + + /// It's possible that we connected to different [Zoo]Keeper instance + /// so we may read a bit stale state. Run some writing request before loading log entries + /// to make that instance up-to-date. + zookeeper->set(zookeeper_path_log, ""); } loadNewEntries(); removeOldEntries(); + tryFinalizeUnknownStateTransactions(); } catch (const Coordination::Exception &) { @@ -241,12 +261,12 @@ void TransactionLog::runUpdatingThread() void TransactionLog::loadNewEntries() { Strings entries_list = zookeeper->getChildren(zookeeper_path_log, nullptr, log_updated_event); - assert(!entries_list.empty()); + chassert(!entries_list.empty()); std::sort(entries_list.begin(), entries_list.end()); auto it = std::upper_bound(entries_list.begin(), entries_list.end(), last_loaded_entry); loadEntries(it, entries_list.end()); - assert(last_loaded_entry == entries_list.back()); - assert(latest_snapshot == deserializeCSN(last_loaded_entry)); + chassert(last_loaded_entry == entries_list.back()); + chassert(latest_snapshot == deserializeCSN(last_loaded_entry)); latest_snapshot.notify_all(); } @@ -309,6 +329,46 @@ void TransactionLog::removeOldEntries() tid_to_csn.erase(tid_hash); } +void TransactionLog::tryFinalizeUnknownStateTransactions() +{ + /// We just recovered connection to [Zoo]Keeper. + /// Check if transactions in unknown state were actually committed or not and finalize or rollback them. + UnknownStateList list; + { + /// We must be sure that the corresponding CSN entry is loaded from ZK. + /// Otherwise we may accidentally rollback committed transaction in case of race condition like this: + /// - runUpdatingThread: loaded some entries, ready to call tryFinalizeUnknownStateTransactions() + /// - commitTransaction: creates CSN entry in the log (txn is committed) + /// - [session expires] + /// - commitTransaction: catches Coordination::Exception (maybe due to fault injection), appends txn to unknown_state_list + /// - runUpdatingThread: calls tryFinalizeUnknownStateTransactions(), fails to find CSN for this txn, rolls it back + /// So all CSN entries that might exist at the moment of appending txn to unknown_state_list + /// must be loaded from ZK before we start finalize that txn. + /// That's why we use two lists here: + /// 1. At first we put txn into unknown_state_list + /// 2. We move it to unknown_state_list_loaded when runUpdatingThread done at least one iteration + /// 3. Then we can safely finalize txns from unknown_state_list_loaded, because all required entries are loaded + std::lock_guard lock{running_list_mutex}; + std::swap(list, unknown_state_list); + std::swap(list, unknown_state_list_loaded); + } + + for (auto & [txn, state_guard] : list) + { + /// CSNs must be already loaded, only need to check if the corresponding mapping exists. + if (auto csn = getCSN(txn->tid)) + { + finalizeCommittedTransaction(txn, csn, state_guard); + } + else + { + assertTIDIsNotOutdated(txn->tid); + state_guard = {}; + rollbackTransaction(txn->shared_from_this()); + } + } +} + CSN TransactionLog::getLatestSnapshot() const { return latest_snapshot.load(); @@ -334,58 +394,117 @@ MergeTreeTransactionPtr TransactionLog::beginTransaction() return txn; } -CSN TransactionLog::commitTransaction(const MergeTreeTransactionPtr & txn) +CSN TransactionLog::commitTransaction(const MergeTreeTransactionPtr & txn, bool throw_on_unknown_status) { /// Some precommit checks, may throw - auto committing_lock = txn->beforeCommit(); + auto state_guard = txn->beforeCommit(); - CSN new_csn; + CSN allocated_csn = Tx::UnknownCSN; if (txn->isReadOnly()) { /// Don't need to allocate CSN in ZK for readonly transactions, it's safe to use snapshot/start_csn as "commit" timestamp LOG_TEST(log, "Closing readonly transaction {}", txn->tid); - new_csn = txn->snapshot; - tryWriteEventToSystemLog(log, global_context, TransactionsInfoLogElement::COMMIT, txn->tid, new_csn); } else { LOG_TEST(log, "Committing transaction {}", txn->dumpDescription()); - /// TODO handle connection loss /// TODO support batching auto current_zookeeper = getZooKeeper(); - String path_created = current_zookeeper->create(zookeeper_path_log + "/csn-", serializeTID(txn->tid), zkutil::CreateMode::PersistentSequential); /// Commit point - NOEXCEPT_SCOPE; + String csn_path_created; + try + { + if (unlikely(fault_probability_before_commit)) + { + std::bernoulli_distribution fault(fault_probability_before_commit); + if (fault(thread_local_rng)) + throw Coordination::Exception("Fault injected (before commit)", Coordination::Error::ZCONNECTIONLOSS); + } + /// Commit point + csn_path_created = current_zookeeper->create(zookeeper_path_log + "/csn-", serializeTID(txn->tid), zkutil::CreateMode::PersistentSequential); + + if (unlikely(fault_probability_after_commit)) + { + std::bernoulli_distribution fault(fault_probability_after_commit); + if (fault(thread_local_rng)) + throw Coordination::Exception("Fault injected (after commit)", Coordination::Error::ZCONNECTIONLOSS); + } + } + catch (const Coordination::Exception & e) + { + if (!Coordination::isHardwareError(e.code)) + throw; + + /// We don't know if transaction has been actually committed or not. + /// The only thing we can do is to postpone its finalization. + { + std::lock_guard lock{running_list_mutex}; + unknown_state_list.emplace_back(txn.get(), std::move(state_guard)); + } + log_updated_event->set(); + if (throw_on_unknown_status) + throw Exception(ErrorCodes::UNKNOWN_STATUS_OF_TRANSACTION, + "Connection lost on attempt to commit transaction {}, will finalize it later: {}", + txn->tid, e.message()); + + LOG_INFO(log, "Connection lost on attempt to commit transaction {}, will finalize it later: {}", txn->tid, e.message()); + return Tx::CommittingCSN; + } + + /// Do not allow exceptions between commit point and the and of transaction finalization + /// (otherwise it may stuck in COMMITTING state holding snapshot). + NOEXCEPT_SCOPE; /// FIXME Transactions: Sequential node numbers in ZooKeeper are Int32, but 31 bit is not enough for production use /// (overflow is possible in a several weeks/months of active usage) - new_csn = deserializeCSN(path_created.substr(zookeeper_path_log.size() + 1)); + allocated_csn = deserializeCSN(csn_path_created.substr(zookeeper_path_log.size() + 1)); + } - LOG_INFO(log, "Transaction {} committed with CSN={}", txn->tid, new_csn); - tryWriteEventToSystemLog(log, global_context, TransactionsInfoLogElement::COMMIT, txn->tid, new_csn); + return finalizeCommittedTransaction(txn.get(), allocated_csn, state_guard); +} - /// Wait for committed changes to become actually visible, so the next transaction in this session will see the changes - /// TODO it's optional, add a setting for this - auto current_latest_snapshot = latest_snapshot.load(); - while (current_latest_snapshot < new_csn && !stop_flag) - { - latest_snapshot.wait(current_latest_snapshot); - current_latest_snapshot = latest_snapshot.load(); - } +CSN TransactionLog::finalizeCommittedTransaction(MergeTreeTransaction * txn, CSN allocated_csn, scope_guard & state_guard) noexcept +{ + chassert(!allocated_csn == txn->isReadOnly()); + if (allocated_csn) + { + LOG_INFO(log, "Transaction {} committed with CSN={}", txn->tid, allocated_csn); + tryWriteEventToSystemLog(log, global_context, TransactionsInfoLogElement::COMMIT, txn->tid, allocated_csn); + } + else + { + /// Transaction was readonly + allocated_csn = txn->snapshot; + tryWriteEventToSystemLog(log, global_context, TransactionsInfoLogElement::COMMIT, txn->tid, allocated_csn); } /// Write allocated CSN, so we will be able to cleanup log in ZK. This method is noexcept. - txn->afterCommit(new_csn); + txn->afterCommit(allocated_csn); + state_guard = {}; { /// Finally we can remove transaction from the list and release the snapshot std::lock_guard lock{running_list_mutex}; + snapshots_in_use.erase(txn->snapshot_in_use_it); bool removed = running_list.erase(txn->tid.getHash()); if (!removed) - throw Exception(ErrorCodes::LOGICAL_ERROR, "I's a bug: TID {} {} doesn't exist", txn->tid.getHash(), txn->tid); - snapshots_in_use.erase(txn->snapshot_in_use_it); + { + LOG_ERROR(log , "I's a bug: TID {} {} doesn't exist", txn->tid.getHash(), txn->tid); + abort(); + } } - return new_csn; + return allocated_csn; +} + +bool TransactionLog::waitForCSNLoaded(CSN csn) const +{ + auto current_latest_snapshot = latest_snapshot.load(); + while (current_latest_snapshot < csn && !stop_flag) + { + latest_snapshot.wait(current_latest_snapshot); + current_latest_snapshot = latest_snapshot.load(); + } + return csn <= current_latest_snapshot; } void TransactionLog::rollbackTransaction(const MergeTreeTransactionPtr & txn) noexcept @@ -395,8 +514,8 @@ void TransactionLog::rollbackTransaction(const MergeTreeTransactionPtr & txn) no if (!txn->rollback()) { - /// Transaction was cancelled concurrently, it's already rolled back. - assert(txn->csn == Tx::RolledBackCSN); + /// Transaction was cancelled or committed concurrently + chassert(txn->csn != Tx::UnknownCSN); return; } @@ -438,8 +557,8 @@ CSN TransactionLog::getCSN(const TIDHash & tid) CSN TransactionLog::getCSNImpl(const TIDHash & tid_hash) const { - assert(tid_hash); - assert(tid_hash != Tx::EmptyTID.getHash()); + chassert(tid_hash); + chassert(tid_hash != Tx::EmptyTID.getHash()); std::lock_guard lock{mutex}; auto it = tid_to_csn.find(tid_hash); @@ -467,6 +586,8 @@ CSN TransactionLog::getOldestSnapshot() const std::lock_guard lock{running_list_mutex}; if (snapshots_in_use.empty()) return getLatestSnapshot(); + chassert(running_list.size() == snapshots_in_use.size()); + chassert(snapshots_in_use.size() < 2 || snapshots_in_use.front() <= *++snapshots_in_use.begin()); return snapshots_in_use.front(); } diff --git a/src/Interpreters/TransactionLog.h b/src/Interpreters/TransactionLog.h index 86584a74c68..a0268ce9b88 100644 --- a/src/Interpreters/TransactionLog.h +++ b/src/Interpreters/TransactionLog.h @@ -97,7 +97,8 @@ public: /// Tries to commit transaction. Returns Commit Sequence Number. /// Throw if transaction was concurrently killed or if some precommit check failed. /// May throw if ZK connection is lost. Transaction status is unknown in this case. - CSN commitTransaction(const MergeTreeTransactionPtr & txn); + /// Returns CommittingCSN if throw_on_unknown_status is false and connection was lost. + CSN commitTransaction(const MergeTreeTransactionPtr & txn, bool throw_on_unknown_status); /// Releases locks that that were acquired by transaction, releases snapshot, removes transaction from the list of active transactions. /// Normally it should not throw, but if it does for some reason (global memory limit exceeded, disk failure, etc) @@ -119,6 +120,12 @@ public: /// Returns copy of list of running transactions. TransactionsList getTransactionsList() const; + /// Waits for provided CSN (and all previous ones) to be loaded from the log. + /// Returns false if waiting was interrupted (e.g. by shutdown) + bool waitForCSNLoaded(CSN csn) const; + + bool isShuttingDown() const { return stop_flag.load(); } + private: void loadLogFromZooKeeper(); void runUpdatingThread(); @@ -127,6 +134,10 @@ private: void loadNewEntries(); void removeOldEntries(); + CSN finalizeCommittedTransaction(MergeTreeTransaction * txn, CSN allocated_csn, scope_guard & state_guard) noexcept; + + void tryFinalizeUnknownStateTransactions(); + static UInt64 deserializeCSN(const String & csn_node_name); static String serializeCSN(CSN csn); static TransactionID deserializeTID(const String & csn_node_content); @@ -159,6 +170,10 @@ private: mutable std::mutex running_list_mutex; /// Transactions that are currently processed TransactionsList running_list; + /// If we lost connection on attempt to create csn- node then we don't know transaction's state. + using UnknownStateList = std::vector>; + UnknownStateList unknown_state_list; + UnknownStateList unknown_state_list_loaded; /// Ordered list of snapshots that are currently used by some transactions. Needed for background cleanup. std::list snapshots_in_use; @@ -175,6 +190,9 @@ private: std::atomic_bool stop_flag = false; ThreadFromGlobalPool updating_thread; + + Float64 fault_probability_before_commit = 0; + Float64 fault_probability_after_commit = 0; }; template diff --git a/src/Interpreters/TransactionVersionMetadata.cpp b/src/Interpreters/TransactionVersionMetadata.cpp index b965ade8d10..36a4fb9cc5b 100644 --- a/src/Interpreters/TransactionVersionMetadata.cpp +++ b/src/Interpreters/TransactionVersionMetadata.cpp @@ -88,8 +88,8 @@ void VersionMetadata::lockRemovalTID(const TransactionID & tid, const Transactio bool VersionMetadata::tryLockRemovalTID(const TransactionID & tid, const TransactionInfoContext & context, TIDHash * locked_by_id) { - assert(!tid.isEmpty()); - assert(!creation_tid.isEmpty()); + chassert(!tid.isEmpty()); + chassert(!creation_tid.isEmpty()); TIDHash removal_lock_value = tid.getHash(); TIDHash expected_removal_lock_value = 0; bool locked = removal_tid_lock.compare_exchange_strong(expected_removal_lock_value, removal_lock_value); @@ -115,7 +115,7 @@ bool VersionMetadata::tryLockRemovalTID(const TransactionID & tid, const Transac void VersionMetadata::unlockRemovalTID(const TransactionID & tid, const TransactionInfoContext & context) { LOG_TEST(log, "Unlocking removal_tid by {}, table: {}, part: {}", tid, context.table.getNameForLogs(), context.part_name); - assert(!tid.isEmpty()); + chassert(!tid.isEmpty()); TIDHash removal_lock_value = tid.getHash(); TIDHash locked_by = removal_tid_lock.load(); @@ -145,7 +145,7 @@ bool VersionMetadata::isRemovalTIDLocked() const void VersionMetadata::setCreationTID(const TransactionID & tid, TransactionInfoContext * context) { /// NOTE ReplicatedMergeTreeSink may add one part multiple times - assert(creation_tid.isEmpty() || creation_tid == tid); + chassert(creation_tid.isEmpty() || creation_tid == tid); creation_tid = tid; if (context) tryWriteEventToSystemLog(log, TransactionsInfoLogElement::ADD_PART, tid, *context); @@ -158,7 +158,7 @@ bool VersionMetadata::isVisible(const MergeTreeTransaction & txn) bool VersionMetadata::isVisible(CSN snapshot_version, TransactionID current_tid) { - assert(!creation_tid.isEmpty()); + chassert(!creation_tid.isEmpty()); CSN creation = creation_csn.load(std::memory_order_relaxed); TIDHash removal_lock = removal_tid_lock.load(std::memory_order_relaxed); CSN removal = removal_csn.load(std::memory_order_relaxed); @@ -166,10 +166,10 @@ bool VersionMetadata::isVisible(CSN snapshot_version, TransactionID current_tid) [[maybe_unused]] bool had_creation_csn = creation; [[maybe_unused]] bool had_removal_tid = removal_lock; [[maybe_unused]] bool had_removal_csn = removal; - assert(!had_removal_csn || had_removal_tid); - assert(!had_removal_csn || had_creation_csn); - assert(creation == Tx::UnknownCSN || creation == Tx::PrehistoricCSN || Tx::MaxReservedCSN < creation); - assert(removal == Tx::UnknownCSN || removal == Tx::PrehistoricCSN || Tx::MaxReservedCSN < removal); + chassert(!had_removal_csn || had_removal_tid); + chassert(!had_removal_csn || had_creation_csn); + chassert(creation == Tx::UnknownCSN || creation == Tx::PrehistoricCSN || Tx::MaxReservedCSN < creation); + chassert(removal == Tx::UnknownCSN || removal == Tx::PrehistoricCSN || Tx::MaxReservedCSN < removal); /// Special snapshot for introspection purposes if (unlikely(snapshot_version == Tx::EverythingVisibleCSN)) @@ -204,8 +204,8 @@ bool VersionMetadata::isVisible(CSN snapshot_version, TransactionID current_tid) /// Data part has creation_tid/removal_tid, but does not have creation_csn/removal_csn. /// It means that some transaction is creating/removing the part right now or has done it recently /// and we don't know if it was already committed or not. - assert(!had_creation_csn || (had_removal_tid && !had_removal_csn)); - assert(current_tid.isEmpty() || (creation_tid != current_tid && removal_lock != current_tid.getHash())); + chassert(!had_creation_csn || (had_removal_tid && !had_removal_csn)); + chassert(current_tid.isEmpty() || (creation_tid != current_tid && removal_lock != current_tid.getHash())); /// Before doing CSN lookup, let's check some extra conditions. /// If snapshot_version <= some_tid.start_csn, then changes of the transaction with some_tid @@ -347,8 +347,8 @@ void VersionMetadata::write(WriteBuffer & buf) const if (removal_tid_lock) { - assert(!removal_tid.isEmpty()); - assert(removal_tid.getHash() == removal_tid_lock); + chassert(!removal_tid.isEmpty()); + chassert(removal_tid.getHash() == removal_tid_lock); writeRemovalTID(buf); writeCSN(buf, REMOVAL, /* internal */ true); } @@ -384,21 +384,23 @@ void VersionMetadata::read(ReadBuffer & buf) if (name == CREATION_CSN_STR) { - assert(!creation_csn); + chassert(!creation_csn); creation_csn = read_csn(); } else if (name == REMOVAL_TID_STR) { /// NOTE Metadata file may actually contain multiple creation TIDs, we need the last one. removal_tid = TransactionID::read(buf); - if (!removal_tid.isEmpty()) + if (removal_tid.isEmpty()) + removal_tid_lock = 0; + else removal_tid_lock = removal_tid.getHash(); } else if (name == REMOVAL_CSN_STR) { if (removal_tid.isEmpty()) throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "Found removal_csn in metadata file, but removal_tid is {}", removal_tid); - assert(!removal_csn); + chassert(!removal_csn); removal_csn = read_csn(); } else diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 3c03bea3dd1..186c8c30cfa 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -444,9 +444,10 @@ static std::tuple executeQueryImpl( if (auto txn = context->getCurrentTransaction()) { - assert(txn->getState() != MergeTreeTransaction::COMMITTED); + chassert(txn->getState() != MergeTreeTransaction::COMMITTING); + chassert(txn->getState() != MergeTreeTransaction::COMMITTED); if (txn->getState() == MergeTreeTransaction::ROLLED_BACK && !ast->as() && !ast->as()) - throw Exception(ErrorCodes::INVALID_TRANSACTION, "Cannot execute query: transaction is rolled back"); + throw Exception(ErrorCodes::INVALID_TRANSACTION, "Cannot execute query because current transaction failed. Expecting ROLLBACK statement."); } /// Interpret SETTINGS clauses as early as possible (before invoking the corresponding interpreter), diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 29cd08554b5..daf27c0dc67 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -105,9 +105,9 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; - bool require_type = true; - bool allow_null_modifiers = false; - bool check_keywords_after_name = false; + const bool require_type = true; + const bool allow_null_modifiers = false; + const bool check_keywords_after_name = false; /// just for ALTER TABLE ALTER COLUMN use bool check_type_keyword = false; }; @@ -175,7 +175,22 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ASTPtr ttl_expression; ASTPtr collation_expression; - if (!s_default.checkWithoutMoving(pos, expected) + auto null_check_without_moving = [&]() -> bool + { + if (!allow_null_modifiers) + return false; + + if (s_null.checkWithoutMoving(pos, expected)) + return true; + + Pos before_null = pos; + bool res = s_not.check(pos, expected) && s_null.checkWithoutMoving(pos, expected); + pos = before_null; + return res; + }; + + if (!null_check_without_moving() + && !s_default.checkWithoutMoving(pos, expected) && !s_materialized.checkWithoutMoving(pos, expected) && !s_ephemeral.checkWithoutMoving(pos, expected) && !s_alias.checkWithoutMoving(pos, expected) @@ -195,6 +210,18 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E } } + if (allow_null_modifiers) + { + if (s_not.check(pos, expected)) + { + if (!s_null.check(pos, expected)) + return false; + null_modifier.emplace(false); + } + else if (s_null.check(pos, expected)) + null_modifier.emplace(true); + } + Pos pos_before_specifier = pos; if (s_default.ignore(pos, expected) || s_materialized.ignore(pos, expected) || s_alias.ignore(pos, expected)) { @@ -230,7 +257,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (require_type && !type && !default_expression) return false; /// reject column name without type - if (type && allow_null_modifiers) + if ((type || default_expression) && allow_null_modifiers && !null_modifier.has_value()) { if (s_not.ignore(pos, expected)) { diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp index da3e3efe807..05fc3b8ca2a 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp @@ -187,7 +187,7 @@ void registerInputFormatArrow(FormatFactory & factory) { return std::make_shared(buf, sample, false, format_settings); }); - factory.markFormatAsColumnOriented("Arrow"); + factory.markFormatSupportsSubsetOfColumns("Arrow"); factory.registerInputFormat( "ArrowStream", [](ReadBuffer & buf, diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 7c5dd2a03ea..c792d828e44 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -36,7 +36,6 @@ #include #include - /// UINT16 and UINT32 are processed separately, see comments in readColumnFromArrowColumn. #define FOR_ARROW_NUMERIC_TYPES(M) \ M(arrow::Type::UINT8, DB::UInt8) \ diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index bd5a6368291..e3cc896466b 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -215,14 +214,16 @@ namespace DB std::unordered_map> & dictionary_values) { const auto * column_tuple = assert_cast(column.get()); - const auto & nested_types = assert_cast(column_type.get())->getElements(); + const auto * type_tuple = assert_cast(column_type.get()); + const auto & nested_types = type_tuple->getElements(); + const auto & nested_names = type_tuple->getElementNames(); arrow::StructBuilder & builder = assert_cast(*array_builder); for (size_t i = 0; i != column_tuple->tupleSize(); ++i) { ColumnPtr nested_column = column_tuple->getColumnPtr(i); - fillArrowArray(column_name + "." + std::to_string(i), nested_column, nested_types[i], null_bytemap, builder.field_builder(i), format_name, start, end, output_string_as_string, dictionary_values); + fillArrowArray(column_name + "." + nested_names[i], nested_column, nested_types[i], null_bytemap, builder.field_builder(i), format_name, start, end, output_string_as_string, dictionary_values); } for (size_t i = start; i != end; ++i) @@ -661,14 +662,15 @@ namespace DB if (isTuple(column_type)) { - const auto & nested_types = assert_cast(column_type.get())->getElements(); + const auto & tuple_type = assert_cast(column_type.get()); + const auto & nested_types = tuple_type->getElements(); + const auto & nested_names = tuple_type->getElementNames(); const auto * tuple_column = assert_cast(column.get()); std::vector> nested_fields; for (size_t i = 0; i != nested_types.size(); ++i) { - String name = column_name + "." + std::to_string(i); - auto nested_arrow_type = getArrowType(nested_types[i], tuple_column->getColumnPtr(i), name, format_name, output_string_as_string, out_is_column_nullable); - nested_fields.push_back(std::make_shared(name, nested_arrow_type, *out_is_column_nullable)); + auto nested_arrow_type = getArrowType(nested_types[i], tuple_column->getColumnPtr(i), nested_names[i], format_name, output_string_as_string, out_is_column_nullable); + nested_fields.push_back(std::make_shared(nested_names[i], nested_arrow_type, *out_is_column_nullable)); } return arrow::struct_(nested_fields); } diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index 9990c33f0bb..0eaa02c97cb 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -112,7 +112,9 @@ String CSVFormatReader::readCSVFieldIntoString() void CSVFormatReader::skipField() { - readCSVFieldIntoString(); + skipWhitespacesAndTabs(*in); + NullOutput out; + readCSVStringInto(out, *in, format_settings.csv); } void CSVFormatReader::skipRowEndDelimiter() @@ -374,6 +376,7 @@ void registerFileSegmentationEngineCSV(FormatFactory & factory) }; registerWithNamesAndTypes("CSV", register_func); + markFormatWithNamesAndTypesSupportsSamplingColumns("CSV", factory); } void registerCSVSchemaReader(FormatFactory & factory) diff --git a/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp b/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp index 67743a04bf3..ad173e449d6 100644 --- a/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp @@ -310,6 +310,7 @@ void registerInputFormatCapnProto(FormatFactory & factory) return std::make_shared(buf, sample, std::move(params), FormatSchemaInfo(settings, "CapnProto", true), settings); }); + factory.markFormatSupportsSubsetOfColumns("CapnProto"); factory.registerFileExtension("capnp", "CapnProto"); } diff --git a/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp index 74c5fb1945a..56a639a0e30 100644 --- a/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp @@ -333,6 +333,7 @@ void registerInputFormatCustomSeparated(FormatFactory & factory) }); }; registerWithNamesAndTypes(ignore_spaces ? "CustomSeparatedIgnoreSpaces" : "CustomSeparated", register_func); + markFormatWithNamesAndTypesSupportsSamplingColumns(ignore_spaces ? "CustomSeparatedIgnoreSpaces" : "CustomSeparated", factory); } } diff --git a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.cpp b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.cpp index 935462a6fe4..22264d01a57 100644 --- a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.cpp @@ -54,6 +54,7 @@ void registerInputFormatJSONColumns(FormatFactory & factory) return std::make_shared(buf, sample, settings, std::make_unique(buf)); } ); + factory.markFormatSupportsSubsetOfColumns("JSONColumns"); } void registerJSONColumnsSchemaReader(FormatFactory & factory) diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp index ef59fc8f05a..0b7cc6669be 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp @@ -229,6 +229,7 @@ void registerInputFormatJSONCompactEachRow(FormatFactory & factory) }; registerWithNamesAndTypes(yield_strings ? "JSONCompactStringsEachRow" : "JSONCompactEachRow", register_func); + markFormatWithNamesAndTypesSupportsSamplingColumns(yield_strings ? "JSONCompactStringsEachRow" : "JSONCompactEachRow", factory); } } diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index 629ed66a1a4..9eef72f95da 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -393,6 +393,11 @@ void registerInputFormatJSONEachRow(FormatFactory & factory) { return std::make_shared(buf, sample, std::move(params), settings, true); }); + + factory.markFormatSupportsSubsetOfColumns("JSONEachRow"); + factory.markFormatSupportsSubsetOfColumns("JSONLines"); + factory.markFormatSupportsSubsetOfColumns("NDJSON"); + factory.markFormatSupportsSubsetOfColumns("JSONStringsEachRow"); } void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory) diff --git a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp index 7768339b064..8e787edf8ab 100644 --- a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp @@ -397,8 +397,8 @@ bool MySQLDumpRowInputFormat::readField(IColumn & column, size_t column_idx) void MySQLDumpRowInputFormat::skipField() { - String tmp; - readQuotedField(tmp, *in); + NullOutput out; + readQuotedFieldInto(out, *in); } MySQLDumpSchemaReader::MySQLDumpSchemaReader(ReadBuffer & in_, const FormatSettings & format_settings_) diff --git a/src/Processors/Formats/Impl/NativeFormat.cpp b/src/Processors/Formats/Impl/NativeFormat.cpp index c1dc60022f5..423fd483712 100644 --- a/src/Processors/Formats/Impl/NativeFormat.cpp +++ b/src/Processors/Formats/Impl/NativeFormat.cpp @@ -15,9 +15,9 @@ namespace DB class NativeInputFormat final : public IInputFormat { public: - NativeInputFormat(ReadBuffer & buf, const Block & header_) + NativeInputFormat(ReadBuffer & buf, const Block & header_, const FormatSettings & settings) : IInputFormat(header_, buf) - , reader(std::make_unique(buf, header_, 0)) + , reader(std::make_unique(buf, header_, 0, settings.skip_unknown_fields)) , header(header_) {} String getName() const override { return "Native"; } @@ -112,10 +112,11 @@ void registerInputFormatNative(FormatFactory & factory) ReadBuffer & buf, const Block & sample, const RowInputFormatParams &, - const FormatSettings &) + const FormatSettings & settings) { - return std::make_shared(buf, sample); + return std::make_shared(buf, sample, settings); }); + factory.markFormatSupportsSubsetOfColumns("Native"); } void registerOutputFormatNative(FormatFactory & factory) diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index 87351b6c5d9..36126c21bf1 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -198,7 +198,7 @@ void registerInputFormatORC(FormatFactory & factory) { return std::make_shared(buf, sample, settings); }); - factory.markFormatAsColumnOriented("ORC"); + factory.markFormatSupportsSubsetOfColumns("ORC"); } void registerORCSchemaReader(FormatFactory & factory) diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index aaa3e8fe976..5e979c3d35a 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -55,7 +55,7 @@ ORCBlockOutputFormat::ORCBlockOutputFormat(WriteBuffer & out_, const Block & hea data_types.push_back(recursiveRemoveLowCardinality(type)); } -ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & type, const std::string & column_name) +ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & type) { switch (type->getTypeId()) { @@ -106,12 +106,12 @@ ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & t } case TypeIndex::Nullable: { - return getORCType(removeNullable(type), column_name); + return getORCType(removeNullable(type)); } case TypeIndex::Array: { const auto * array_type = assert_cast(type.get()); - return orc::createListType(getORCType(array_type->getNestedType(), column_name)); + return orc::createListType(getORCType(array_type->getNestedType())); } case TypeIndex::Decimal32: { @@ -131,21 +131,19 @@ ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & t case TypeIndex::Tuple: { const auto * tuple_type = assert_cast(type.get()); + const auto & nested_names = tuple_type->getElementNames(); const auto & nested_types = tuple_type->getElements(); auto struct_type = orc::createStructType(); for (size_t i = 0; i < nested_types.size(); ++i) - { - String name = column_name + "." + std::to_string(i); - struct_type->addStructField(name, getORCType(nested_types[i], name)); - } + struct_type->addStructField(nested_names[i], getORCType(nested_types[i])); return struct_type; } case TypeIndex::Map: { const auto * map_type = assert_cast(type.get()); return orc::createMapType( - getORCType(map_type->getKeyType(), column_name), - getORCType(map_type->getValueType(), column_name) + getORCType(map_type->getKeyType()), + getORCType(map_type->getValueType()) ); } default: @@ -514,7 +512,7 @@ void ORCBlockOutputFormat::prepareWriter() options.setCompression(orc::CompressionKind::CompressionKind_NONE); size_t columns_count = header.columns(); for (size_t i = 0; i != columns_count; ++i) - schema->addStructField(header.safeGetByPosition(i).name, getORCType(recursiveRemoveLowCardinality(data_types[i]), header.safeGetByPosition(i).name)); + schema->addStructField(header.safeGetByPosition(i).name, getORCType(recursiveRemoveLowCardinality(data_types[i]))); writer = orc::createWriter(*schema, &output_stream, options); } diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h index f69fd1c0aab..d4a19353915 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h @@ -42,7 +42,7 @@ private: void consume(Chunk chunk) override; void finalizeImpl() override; - ORC_UNIQUE_PTR getORCType(const DataTypePtr & type, const std::string & column_name); + ORC_UNIQUE_PTR getORCType(const DataTypePtr & type); /// ConvertFunc is needed for type UInt8, because firstly UInt8 (char8_t) must be /// converted to unsigned char (bugprone-signed-char-misuse in clang). diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 062f161b7f9..12fa9710c42 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -192,7 +192,7 @@ void registerInputFormatParquet(FormatFactory & factory) { return std::make_shared(buf, sample, settings); }); - factory.markFormatAsColumnOriented("Parquet"); + factory.markFormatSupportsSubsetOfColumns("Parquet"); } void registerParquetSchemaReader(FormatFactory & factory) diff --git a/src/Processors/Formats/Impl/ProtobufListInputFormat.cpp b/src/Processors/Formats/Impl/ProtobufListInputFormat.cpp index 6fbcaa15536..4599734591f 100644 --- a/src/Processors/Formats/Impl/ProtobufListInputFormat.cpp +++ b/src/Processors/Formats/Impl/ProtobufListInputFormat.cpp @@ -79,7 +79,7 @@ void registerInputFormatProtobufList(FormatFactory & factory) return std::make_shared(buf, sample, std::move(params), FormatSchemaInfo(settings, "Protobuf", true), settings.protobuf.input_flatten_google_wrappers); }); - factory.markFormatAsColumnOriented("ProtobufList"); + factory.markFormatSupportsSubsetOfColumns("ProtobufList"); } void registerProtobufListSchemaReader(FormatFactory & factory) diff --git a/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp b/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp index 5c953a3fcc9..0376bf2c292 100644 --- a/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp @@ -69,6 +69,7 @@ void registerInputFormatProtobuf(FormatFactory & factory) with_length_delimiter, settings.protobuf.input_flatten_google_wrappers); }); + factory.markFormatSupportsSubsetOfColumns(with_length_delimiter ? "Protobuf" : "ProtobufSingle"); } } diff --git a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp index 5c48062ace8..fe2c0c5ecdd 100644 --- a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp @@ -277,6 +277,8 @@ void registerInputFormatTSKV(FormatFactory & factory) { return std::make_shared(buf, sample, std::move(params), settings); }); + + factory.markFormatSupportsSubsetOfColumns("TSKV"); } void registerTSKVSchemaReader(FormatFactory & factory) { diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 5f39c7bd646..0be8257f463 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -80,7 +80,11 @@ String TabSeparatedFormatReader::readFieldIntoString() void TabSeparatedFormatReader::skipField() { - readFieldIntoString(); + NullOutput out; + if (is_raw) + readStringInto(out, *in); + else + readEscapedStringInto(out, *in); } void TabSeparatedFormatReader::skipHeaderRow() @@ -347,6 +351,8 @@ void registerFileSegmentationEngineTabSeparated(FormatFactory & factory) registerWithNamesAndTypes(is_raw ? "TSVRaw" : "TSV", register_func); registerWithNamesAndTypes(is_raw ? "TabSeparatedRaw" : "TabSeparated", register_func); + markFormatWithNamesAndTypesSupportsSamplingColumns(is_raw ? "TSVRaw" : "TSV", factory); + markFormatWithNamesAndTypesSupportsSamplingColumns(is_raw ? "TabSeparatedRaw" : "TabSeparated", factory); } // We can use the same segmentation engine for TSKV. diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 9b7eed9f5ee..2625bf38bf7 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -77,8 +77,7 @@ static size_t tryAddNewFilterStep( /// New filter column is the first one. auto split_filter_column_name = (*split_filter->getIndex().begin())->result_name; node.step = std::make_unique( - node.children.at(0)->step->getOutputStream(), - std::move(split_filter), std::move(split_filter_column_name), true); + node.children.at(0)->step->getOutputStream(), std::move(split_filter), std::move(split_filter_column_name), true); return 3; } @@ -194,13 +193,13 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes /// Push down is for left table only. We need to update JoinStep for push down into right. /// Only inner and left join are supported. Other types may generate default values for left table keys. /// So, if we push down a condition like `key != 0`, not all rows may be filtered. - if (table_join.oneDisjunct() && (table_join.kind() == ASTTableJoin::Kind::Inner || table_join.kind() == ASTTableJoin::Kind::Left)) + if (table_join.kind() == ASTTableJoin::Kind::Inner || table_join.kind() == ASTTableJoin::Kind::Left) { const auto & left_header = join->getInputStreams().front().header; const auto & res_header = join->getOutputStream().header; Names allowed_keys; - const auto & key_names_left = table_join.getOnlyClause().key_names_left; - for (const auto & name : key_names_left) + const auto & source_columns = left_header.getNames(); + for (const auto & name : source_columns) { /// Skip key if it is renamed. /// I don't know if it is possible. Just in case. diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index 9a8dd151830..ff30cfd8cf3 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -34,7 +34,7 @@ void optimizeTree(const QueryPlanOptimizationSettings & settings, QueryPlan::Nod }; std::stack stack; - stack.push(Frame{.node = &root}); + stack.push({.node = &root}); size_t max_optimizations_to_apply = settings.max_optimizations_to_apply; size_t total_applied_optimizations = 0; @@ -50,10 +50,10 @@ void optimizeTree(const QueryPlanOptimizationSettings & settings, QueryPlan::Nod /// Traverse all children first. if (frame.next_child < frame.node->children.size()) { - stack.push(Frame + stack.push( { - .node = frame.node->children[frame.next_child], - .depth_limit = frame.depth_limit ? (frame.depth_limit - 1) : 0, + .node = frame.node->children[frame.next_child], + .depth_limit = frame.depth_limit ? (frame.depth_limit - 1) : 0, }); ++frame.next_child; diff --git a/src/Processors/Transforms/PartialSortingTransform.cpp b/src/Processors/Transforms/PartialSortingTransform.cpp index 6a787a6cd15..3687fa770f0 100644 --- a/src/Processors/Transforms/PartialSortingTransform.cpp +++ b/src/Processors/Transforms/PartialSortingTransform.cpp @@ -81,9 +81,7 @@ size_t getFilterMask(const ColumnRawPtrs & lhs, const ColumnRawPtrs & rhs, size_ { /// Leave only rows that are less then row from rhs. filter[i] = compare_results[i] < 0; - - if (filter[i]) - ++result_size_hint; + result_size_hint += filter[i]; } return result_size_hint; diff --git a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp index 07584075097..c8ae9c6e07c 100644 --- a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp +++ b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp @@ -53,11 +53,20 @@ void WriteBufferFromHTTPServerResponse::writeHeaderProgress() *response_header_ostr << "X-ClickHouse-Progress: " << progress_string_writer.str() << "\r\n" << std::flush; } +void WriteBufferFromHTTPServerResponse::writeExceptionCode() +{ + if (headers_finished_sending || !exception_code) + return; + if (response_header_ostr) + *response_header_ostr << "X-ClickHouse-Exception-Code: " << exception_code << "\r\n" << std::flush; +} + void WriteBufferFromHTTPServerResponse::finishSendHeaders() { if (!headers_finished_sending) { writeHeaderSummary(); + writeExceptionCode(); headers_finished_sending = true; if (!is_http_method_head) @@ -150,7 +159,7 @@ void WriteBufferFromHTTPServerResponse::onProgress(const Progress & progress) accumulated_progress.incrementPiecewiseAtomically(progress); - if (progress_watch.elapsed() >= send_progress_interval_ms * 1000000) + if (send_progress && progress_watch.elapsed() >= send_progress_interval_ms * 1000000) { progress_watch.restart(); diff --git a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.h b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.h index 9f1d3e897e3..6905d5df8b5 100644 --- a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.h +++ b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.h @@ -66,12 +66,17 @@ public: add_cors_header = enable_cors; } + /// Send progress + void setSendProgress(bool send_progress_) { send_progress = send_progress_; } + /// Don't send HTTP headers with progress more frequently. void setSendProgressInterval(size_t send_progress_interval_ms_) { send_progress_interval_ms = send_progress_interval_ms_; } + void setExceptionCode(int exception_code_) { exception_code = exception_code_; } + private: /// Send at least HTTP headers if no data has been sent yet. /// Use after the data has possibly been sent and no error happened (and thus you do not plan @@ -88,6 +93,8 @@ private: void writeHeaderProgress(); // Used for write the header X-ClickHouse-Summary void writeHeaderSummary(); + // Use to write the header X-ClickHouse-Exception-Code even when progress has been sent + void writeExceptionCode(); /// This method finish headers with \r\n, allowing to start to send body. void finishSendHeaders(); @@ -113,9 +120,12 @@ private: bool headers_finished_sending = false; /// If true, you could not add any headers. Progress accumulated_progress; + bool send_progress = false; size_t send_progress_interval_ms = 100; Stopwatch progress_watch; + int exception_code = 0; + std::mutex mutex; /// progress callback could be called from different threads. }; diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 4a1de7ddf2d..39870fc91dc 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -103,6 +103,8 @@ namespace ErrorCodes extern const int INVALID_SESSION_TIMEOUT; extern const int HTTP_LENGTH_REQUIRED; extern const int SUPPORT_IS_DISABLED; + + extern const int TIMEOUT_EXCEEDED; } namespace @@ -228,6 +230,10 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti { return HTTPResponse::HTTP_LENGTH_REQUIRED; } + else if (exception_code == ErrorCodes::TIMEOUT_EXCEEDED) + { + return HTTPResponse::HTTP_REQUEST_TIMEOUT; + } return HTTPResponse::HTTP_INTERNAL_SERVER_ERROR; } @@ -771,6 +777,7 @@ void HTTPHandler::processQuery( if (client_supports_http_compression) used_output.out->setCompressionLevel(settings.http_zlib_compression_level); + used_output.out->setSendProgress(settings.send_progress_in_http_headers); used_output.out->setSendProgressInterval(settings.http_headers_progress_interval_ms); /// If 'http_native_compression_disable_checksumming_on_decompress' setting is turned on, @@ -803,8 +810,8 @@ void HTTPHandler::processQuery( }; /// While still no data has been sent, we will report about query execution progress by sending HTTP headers. - if (settings.send_progress_in_http_headers) - append_callback([&used_output] (const Progress & progress) { used_output.out->onProgress(progress); }); + /// Note that we add it unconditionally so the progress is available for `X-ClickHouse-Summary` + append_callback([&used_output](const Progress & progress) { used_output.out->onProgress(progress); }); if (settings.readonly > 0 && settings.cancel_http_readonly_queries_on_client_close) { @@ -843,7 +850,12 @@ void HTTPHandler::trySendExceptionToClient( const std::string & s, int exception_code, HTTPServerRequest & request, HTTPServerResponse & response, Output & used_output) try { - response.set("X-ClickHouse-Exception-Code", toString(exception_code)); + /// In case data has already been sent, like progress headers, try using the output buffer to + /// set the exception code since it will be able to append it if it hasn't finished writing headers + if (response.sent() && used_output.out) + used_output.out->setExceptionCode(exception_code); + else + response.set("X-ClickHouse-Exception-Code", toString(exception_code)); /// FIXME: make sure that no one else is reading from the same stream at the moment. diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index d114bb67016..5799100664e 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -476,9 +476,9 @@ private: }; -bool StorageHDFS::isColumnOriented() const +bool StorageHDFS::supportsSubsetOfColumns() const { - return format_name != "Distributed" && FormatFactory::instance().checkIfFormatIsColumnOriented(format_name); + return format_name != "Distributed" && FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(format_name); } Pipe StorageHDFS::read( @@ -527,7 +527,7 @@ Pipe StorageHDFS::read( ColumnsDescription columns_description; Block block_for_format; - if (isColumnOriented()) + if (supportsSubsetOfColumns()) { auto fetch_columns = column_names; const auto & virtuals = getVirtuals(); diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index b431407eba5..c8ebbfcfaac 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -57,7 +57,7 @@ public: /// Is is useful because column oriented formats could effectively skip unknown columns /// So we can create a header of only required columns in read method and ask /// format to read only them. Note: this hack cannot be done with ordinary formats like TSV. - bool isColumnOriented() const override; + bool supportsSubsetOfColumns() const override; static ColumnsDescription getTableStructureFromData( const String & format, diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index c66e1acc6e5..38c8c054a9b 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -668,7 +668,7 @@ HiveFilePtr StorageHive::getHiveFileIfNeeded( return hive_file; } -bool StorageHive::isColumnOriented() const +bool StorageHive::supportsSubsetOfColumns() const { return format_name == "Parquet" || format_name == "ORC"; } @@ -822,7 +822,7 @@ std::optional StorageHive::totalRowsImpl(const Settings & settings, const SelectQueryInfo & query_info, ContextPtr context_, PruneLevel prune_level) const { /// Row-based format like Text doesn't support totalRowsByPartitionPredicate - if (!isColumnOriented()) + if (!supportsSubsetOfColumns()) return {}; auto hive_metastore_client = HiveMetastoreClientFactory::instance().getOrCreate(hive_metastore_url); diff --git a/src/Storages/Hive/StorageHive.h b/src/Storages/Hive/StorageHive.h index 1b37a0afd15..d92d2dbd745 100644 --- a/src/Storages/Hive/StorageHive.h +++ b/src/Storages/Hive/StorageHive.h @@ -63,7 +63,7 @@ public: NamesAndTypesList getVirtuals() const override; - bool isColumnOriented() const override; + bool supportsSubsetOfColumns() const override; std::optional totalRows(const Settings & settings) const override; std::optional totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, ContextPtr context_) const override; diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 005229acf4b..ed17a3af972 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -585,7 +585,7 @@ public: /// Returns true if all disks of storage are read-only. virtual bool isStaticStorage() const; - virtual bool isColumnOriented() const { return false; } + virtual bool supportsSubsetOfColumns() const { return false; } /// If it is possible to quickly determine exact number of rows in the table at this moment of time, then return it. /// Used for: diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 09c8fe3a6ab..78896d74d09 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -140,6 +140,16 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedSend}; + { + auto disk = part->volume->getDisk(); + UInt64 revision = parse(params.get("disk_revision", "0")); + if (revision) + disk->syncRevision(revision); + revision = disk->getRevision(); + if (revision) + response.addCookie({"disk_revision", toString(revision)}); + } + if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE) writeBinary(part->checksums.getTotalSizeOnDisk(), out); @@ -418,6 +428,13 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( {"compress", "false"} }); + if (disk) + { + UInt64 revision = disk->getRevision(); + if (revision) + uri.addQueryParameter("disk_revision", toString(revision)); + } + Strings capability; if (try_zero_copy && data_settings->allow_remote_fs_zero_copy_replication) { @@ -452,29 +469,28 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( creds.setPassword(password); } - PooledReadWriteBufferFromHTTP in{ + std::unique_ptr in = std::make_unique( uri, Poco::Net::HTTPRequest::HTTP_POST, - {}, + nullptr, timeouts, creds, DBMS_DEFAULT_BUFFER_SIZE, 0, /* no redirects */ - data_settings->replicated_max_parallel_fetches_for_host - }; + static_cast(data_settings->replicated_max_parallel_fetches_for_host)); - int server_protocol_version = parse(in.getResponseCookie("server_protocol_version", "0")); + int server_protocol_version = parse(in->getResponseCookie("server_protocol_version", "0")); ReservationPtr reservation; size_t sum_files_size = 0; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE) { - readBinary(sum_files_size, in); + readBinary(sum_files_size, *in); if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS) { IMergeTreeDataPart::TTLInfos ttl_infos; String ttl_infos_string; - readBinary(ttl_infos_string, in); + readBinary(ttl_infos_string, *in); ReadBufferFromString ttl_infos_buffer(ttl_infos_string); assertString("ttl format version: 1\n", ttl_infos_buffer); ttl_infos.read(ttl_infos_buffer); @@ -502,18 +518,22 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( if (!disk) disk = reservation->getDisk(); + UInt64 revision = parse(in.getResponseCookie("disk_revision", "0")); + if (revision) + disk->syncRevision(revision); + bool sync = (data_settings->min_compressed_bytes_to_fsync_after_fetch && sum_files_size >= data_settings->min_compressed_bytes_to_fsync_after_fetch); String part_type = "Wide"; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE) - readStringBinary(part_type, in); + readStringBinary(part_type, *in); UUID part_uuid = UUIDHelpers::Nil; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID) - readUUIDText(part_uuid, in); + readUUIDText(part_uuid, *in); - String remote_fs_metadata = parse(in.getResponseCookie("remote_fs_metadata", "")); + String remote_fs_metadata = parse(in->getResponseCookie("remote_fs_metadata", "")); if (!remote_fs_metadata.empty()) { if (!try_zero_copy) @@ -527,7 +547,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( try { - return downloadPartToDiskRemoteMeta(part_name, replica_path, to_detached, tmp_prefix_, disk, in, throttler); + return downloadPartToDiskRemoteMeta(part_name, replica_path, to_detached, tmp_prefix_, disk, *in, throttler); } catch (const Exception & e) { @@ -535,6 +555,18 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( throw; LOG_WARNING(log, fmt::runtime(e.message() + " Will retry fetching part without zero-copy.")); + + /// It's important to release session from HTTP pool. Otherwise it's possible to get deadlock + /// on http pool. + try + { + in.reset(); + } + catch (...) + { + tryLogCurrentException(log); + } + /// Try again but without zero-copy return fetchPart(metadata_snapshot, context, part_name, replica_path, host, port, timeouts, user, password, interserver_scheme, throttler, to_detached, tmp_prefix_, nullptr, false, disk); @@ -548,16 +580,16 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( part_info.partition_id, part_name, new_part_path, replica_path, uri, to_detached, sum_files_size); - in.setNextCallback(ReplicatedFetchReadCallback(*entry)); + in->setNextCallback(ReplicatedFetchReadCallback(*entry)); size_t projections = 0; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION) - readBinary(projections, in); + readBinary(projections, *in); MergeTreeData::DataPart::Checksums checksums; return part_type == "InMemory" - ? downloadPartToMemory(part_name, part_uuid, metadata_snapshot, context, disk, in, projections, throttler) - : downloadPartToDisk(part_name, replica_path, to_detached, tmp_prefix_, sync, disk, in, projections, checksums, throttler); + ? downloadPartToMemory(part_name, part_uuid, metadata_snapshot, context, disk, *in, projections, throttler) + : downloadPartToDisk(part_name, replica_path, to_detached, tmp_prefix_, sync, disk, *in, projections, checksums, throttler); } MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 2c9dd2b4934..40fba34cd03 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1282,12 +1282,12 @@ void IMergeTreeDataPart::storeVersionMetadata() const void IMergeTreeDataPart::appendCSNToVersionMetadata(VersionMetadata::WhichCSN which_csn) const { - assert(!version.creation_tid.isEmpty()); - assert(!(which_csn == VersionMetadata::WhichCSN::CREATION && version.creation_tid.isPrehistoric())); - assert(!(which_csn == VersionMetadata::WhichCSN::CREATION && version.creation_csn == 0)); - assert(!(which_csn == VersionMetadata::WhichCSN::REMOVAL && (version.removal_tid.isPrehistoric() || version.removal_tid.isEmpty()))); - assert(!(which_csn == VersionMetadata::WhichCSN::REMOVAL && version.removal_csn == 0)); - assert(isStoredOnDisk()); + chassert(!version.creation_tid.isEmpty()); + chassert(!(which_csn == VersionMetadata::WhichCSN::CREATION && version.creation_tid.isPrehistoric())); + chassert(!(which_csn == VersionMetadata::WhichCSN::CREATION && version.creation_csn == 0)); + chassert(!(which_csn == VersionMetadata::WhichCSN::REMOVAL && (version.removal_tid.isPrehistoric() || version.removal_tid.isEmpty()))); + chassert(!(which_csn == VersionMetadata::WhichCSN::REMOVAL && version.removal_csn == 0)); + chassert(isStoredOnDisk()); /// Small enough appends to file are usually atomic, /// so we append new metadata instead of rewriting file to reduce number of fsyncs. @@ -1303,10 +1303,10 @@ void IMergeTreeDataPart::appendCSNToVersionMetadata(VersionMetadata::WhichCSN wh void IMergeTreeDataPart::appendRemovalTIDToVersionMetadata(bool clear) const { - assert(!version.creation_tid.isEmpty()); - assert(version.removal_csn == 0); - assert(!version.removal_tid.isEmpty()); - assert(isStoredOnDisk()); + chassert(!version.creation_tid.isEmpty()); + chassert(version.removal_csn == 0); + chassert(!version.removal_tid.isEmpty()); + chassert(isStoredOnDisk()); if (version.creation_tid.isPrehistoric() && !clear) { @@ -1437,7 +1437,9 @@ bool IMergeTreeDataPart::assertHasValidVersionMetadata() const bool valid_removal_tid = version.removal_tid == file.removal_tid || version.removal_tid == Tx::PrehistoricTID; bool valid_creation_csn = version.creation_csn == file.creation_csn || version.creation_csn == Tx::RolledBackCSN; bool valid_removal_csn = version.removal_csn == file.removal_csn || version.removal_csn == Tx::PrehistoricCSN; - if (!valid_creation_tid || !valid_removal_tid || !valid_creation_csn || !valid_removal_csn) + bool valid_removal_tid_lock = (version.removal_tid.isEmpty() && version.removal_tid_lock == 0) + || (version.removal_tid_lock == version.removal_tid.getHash()); + if (!valid_creation_tid || !valid_removal_tid || !valid_creation_csn || !valid_removal_csn || !valid_removal_tid_lock) throw Exception(ErrorCodes::CORRUPTED_DATA, "Invalid version metadata file"); return true; } @@ -1445,7 +1447,8 @@ bool IMergeTreeDataPart::assertHasValidVersionMetadata() const { WriteBufferFromOwnString expected; version.write(expected); - tryLogCurrentException(storage.log, fmt::format("File {} contains:\n{}\nexpected:\n{}", version_file_name, content, expected.str())); + tryLogCurrentException(storage.log, fmt::format("File {} contains:\n{}\nexpected:\n{}\nlock: {}", + version_file_name, content, expected.str(), version.removal_tid_lock)); return false; } } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 50811daa4ab..d2c757f6750 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1364,7 +1364,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) /// Check if CSNs were witten after committing transaction, update and write if needed. bool version_updated = false; - assert(!version.creation_tid.isEmpty()); + chassert(!version.creation_tid.isEmpty()); if (!part->version.creation_csn) { auto min = TransactionLog::getCSN(version.creation_tid); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 47e32337dfe..9dadab56eb5 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -316,9 +316,9 @@ ColumnsDescription StorageFile::getTableStructureFromFile( return readSchemaFromFormat(format, format_settings, read_buffer_iterator, paths.size() > 1, context); } -bool StorageFile::isColumnOriented() const +bool StorageFile::supportsSubsetOfColumns() const { - return format_name != "Distributed" && FormatFactory::instance().checkIfFormatIsColumnOriented(format_name); + return format_name != "Distributed" && FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(format_name); } StorageFile::StorageFile(int table_fd_, CommonArguments args) @@ -465,7 +465,7 @@ public: const ColumnsDescription & columns_description, const FilesInfoPtr & files_info) { - if (storage->isColumnOriented()) + if (storage->supportsSubsetOfColumns()) return storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); else return getHeader(storage_snapshot->metadata, files_info->need_path_column, files_info->need_file_column); @@ -530,7 +530,7 @@ public: auto get_block_for_format = [&]() -> Block { - if (storage->isColumnOriented()) + if (storage->supportsSubsetOfColumns()) return storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); return storage_snapshot->metadata->getSampleBlock(); }; @@ -690,7 +690,7 @@ Pipe StorageFile::read( { const auto get_columns_for_format = [&]() -> ColumnsDescription { - if (isColumnOriented()) + if (supportsSubsetOfColumns()) return storage_snapshot->getDescriptionForColumns(column_names); else return storage_snapshot->metadata->getColumns(); diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 35ab185b14d..f47f6172c1c 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -69,11 +69,11 @@ public: static Strings getPathsList(const String & table_path, const String & user_files_path, ContextPtr context, size_t & total_bytes_to_read); - /// Check if the format is column-oriented. - /// Is is useful because column oriented formats could effectively skip unknown columns + /// Check if the format supports reading only some subset of columns. + /// Is is useful because such formats could effectively skip unknown columns /// So we can create a header of only required columns in read method and ask /// format to read only them. Note: this hack cannot be done with ordinary formats like TSV. - bool isColumnOriented() const override; + bool supportsSubsetOfColumns() const override; bool supportsPartitionBy() const override { return true; } diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index ed7e0c70dbc..7960b7dfac0 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -75,17 +76,27 @@ namespace ErrorCodes extern const int UNEXPECTED_EXPRESSION; extern const int DATABASE_ACCESS_DENIED; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; + extern const int NOT_IMPLEMENTED; } class IOutputFormat; using OutputFormatPtr = std::shared_ptr; -class StorageS3Source::DisclosedGlobIterator::Impl +class StorageS3Source::DisclosedGlobIterator::Impl : WithContext { public: - Impl(const Aws::S3::S3Client & client_, const S3::URI & globbed_uri_) - : client(client_), globbed_uri(globbed_uri_) + Impl( + const Aws::S3::S3Client & client_, + const S3::URI & globbed_uri_, + ASTPtr & query_, + const Block & virtual_header_, + ContextPtr context_) + : WithContext(context_) + , client(client_) + , globbed_uri(globbed_uri_) + , query(query_) + , virtual_header(virtual_header_) { if (globbed_uri.bucket.find_first_of("*?{") != globbed_uri.bucket.npos) throw Exception("Expression can not have wildcards inside bucket name", ErrorCodes::UNEXPECTED_EXPRESSION); @@ -101,6 +112,20 @@ public: return; } + /// Create a virtual block with one row to construct filter + if (query && virtual_header) + { + /// Append "key" column as the filter result + virtual_header.insert({ColumnString::create(), std::make_shared(), "_key"}); + + auto block = virtual_header.cloneEmpty(); + MutableColumns columns = block.mutateColumns(); + for (auto & column : columns) + column->insertDefault(); + block.setColumns(std::move(columns)); + VirtualColumnUtils::prepareFilterBlockWithQuery(query, getContext(), block, filter_ast); + } + request.SetBucket(globbed_uri.bucket); request.SetPrefix(key_prefix); matcher = std::make_unique(makeRegexpPatternFromGlobs(globbed_uri.key)); @@ -144,13 +169,52 @@ private: const auto & result_batch = outcome.GetResult().GetContents(); - buffer.reserve(result_batch.size()); - for (const auto & row : result_batch) + if (filter_ast) { - String key = row.GetKey(); - if (re2::RE2::FullMatch(key, *matcher)) - buffer.emplace_back(std::move(key)); + auto block = virtual_header.cloneEmpty(); + MutableColumnPtr path_column; + MutableColumnPtr file_column; + MutableColumnPtr key_column = block.getByName("_key").column->assumeMutable(); + + if (block.has("_path")) + path_column = block.getByName("_path").column->assumeMutable(); + + if (block.has("_file")) + file_column = block.getByName("_file").column->assumeMutable(); + + for (const auto & row : result_batch) + { + const String & key = row.GetKey(); + if (re2::RE2::FullMatch(key, *matcher)) + { + String path = fs::path(globbed_uri.bucket) / key; + String file = path.substr(path.find_last_of('/') + 1); + if (path_column) + path_column->insert(path); + if (file_column) + file_column->insert(file); + key_column->insert(key); + } + } + + VirtualColumnUtils::filterBlockWithQuery(query, block, getContext(), filter_ast); + const ColumnString & keys = typeid_cast(*block.getByName("_key").column); + size_t rows = block.rows(); + buffer.reserve(rows); + for (size_t i = 0; i < rows; ++i) + buffer.emplace_back(keys.getDataAt(i).toString()); } + else + { + buffer.reserve(result_batch.size()); + for (const auto & row : result_batch) + { + String key = row.GetKey(); + if (re2::RE2::FullMatch(key, *matcher)) + buffer.emplace_back(std::move(key)); + } + } + /// Set iterator only after the whole batch is processed buffer_iter = buffer.begin(); @@ -165,25 +229,83 @@ private: Strings::iterator buffer_iter; Aws::S3::S3Client client; S3::URI globbed_uri; + ASTPtr query; + Block virtual_header; + ASTPtr filter_ast; Aws::S3::Model::ListObjectsV2Request request; Aws::S3::Model::ListObjectsV2Outcome outcome; std::unique_ptr matcher; bool is_finished{false}; }; -StorageS3Source::DisclosedGlobIterator::DisclosedGlobIterator(const Aws::S3::S3Client & client_, const S3::URI & globbed_uri_) - : pimpl(std::make_shared(client_, globbed_uri_)) {} +StorageS3Source::DisclosedGlobIterator::DisclosedGlobIterator( + const Aws::S3::S3Client & client_, const S3::URI & globbed_uri_, ASTPtr query, const Block & virtual_header, ContextPtr context) + : pimpl(std::make_shared(client_, globbed_uri_, query, virtual_header, context)) +{ +} String StorageS3Source::DisclosedGlobIterator::next() { return pimpl->next(); } -class StorageS3Source::KeysIterator::Impl +class StorageS3Source::KeysIterator::Impl : WithContext { public: - explicit Impl(const std::vector & keys_) : keys(keys_) + explicit Impl( + const std::vector & keys_, const String & bucket_, ASTPtr query_, const Block & virtual_header_, ContextPtr context_) + : WithContext(context_), keys(keys_), bucket(bucket_), query(query_), virtual_header(virtual_header_) { + /// Create a virtual block with one row to construct filter + if (query && virtual_header) + { + /// Append "key" column as the filter result + virtual_header.insert({ColumnString::create(), std::make_shared(), "_key"}); + + auto block = virtual_header.cloneEmpty(); + MutableColumns columns = block.mutateColumns(); + for (auto & column : columns) + column->insertDefault(); + block.setColumns(std::move(columns)); + + ASTPtr filter_ast; + VirtualColumnUtils::prepareFilterBlockWithQuery(query, getContext(), block, filter_ast); + + if (filter_ast) + { + block = virtual_header.cloneEmpty(); + MutableColumnPtr path_column; + MutableColumnPtr file_column; + MutableColumnPtr key_column = block.getByName("_key").column->assumeMutable(); + + if (block.has("_path")) + path_column = block.getByName("_path").column->assumeMutable(); + + if (block.has("_file")) + file_column = block.getByName("_file").column->assumeMutable(); + + for (const auto & key : keys) + { + String path = fs::path(bucket) / key; + String file = path.substr(path.find_last_of('/') + 1); + if (path_column) + path_column->insert(path); + if (file_column) + file_column->insert(file); + key_column->insert(key); + } + + VirtualColumnUtils::filterBlockWithQuery(query, block, getContext(), filter_ast); + const ColumnString & keys_col = typeid_cast(*block.getByName("_key").column); + size_t rows = block.rows(); + Strings filtered_keys; + filtered_keys.reserve(rows); + for (size_t i = 0; i < rows; ++i) + filtered_keys.emplace_back(keys_col.getDataAt(i).toString()); + + keys = std::move(filtered_keys); + } + } } String next() @@ -197,9 +319,15 @@ public: private: Strings keys; std::atomic_size_t index = 0; + + String bucket; + ASTPtr query; + Block virtual_header; }; -StorageS3Source::KeysIterator::KeysIterator(const std::vector & keys_) : pimpl(std::make_shared(keys_)) +StorageS3Source::KeysIterator::KeysIterator( + const std::vector & keys_, const String & bucket_, ASTPtr query, const Block & virtual_header, ContextPtr context) + : pimpl(std::make_shared(keys_, bucket_, query, virtual_header, context)) { } @@ -638,6 +766,8 @@ StorageS3::StorageS3( auto columns = storage_metadata.getSampleBlock().getNamesAndTypesList(); virtual_columns = getVirtualsForStorage(columns, default_virtuals); + for (const auto & column : virtual_columns) + virtual_block.insert({column.type->createColumn(), column.type, column.name}); } std::shared_ptr StorageS3::createFileIterator( @@ -646,6 +776,8 @@ std::shared_ptr StorageS3::createFileIterator( bool is_key_with_globs, bool distributed_processing, ContextPtr local_context, + ASTPtr query, + const Block & virtual_block, const std::vector & read_tasks) { if (distributed_processing) @@ -659,36 +791,37 @@ std::shared_ptr StorageS3::createFileIterator( else if (is_key_with_globs) { /// Iterate through disclosed globs and make a source for each file - auto glob_iterator = std::make_shared(*s3_configuration.client, s3_configuration.uri); - return std::make_shared([glob_iterator]() - { - return glob_iterator->next(); - }); + auto glob_iterator = std::make_shared( + *s3_configuration.client, s3_configuration.uri, query, virtual_block, local_context); + return std::make_shared([glob_iterator]() { return glob_iterator->next(); }); } else { - auto keys_iterator = std::make_shared(keys); - return std::make_shared([keys_iterator]() - { - return keys_iterator->next(); - }); + auto keys_iterator + = std::make_shared(keys, s3_configuration.uri.bucket, query, virtual_block, local_context); + return std::make_shared([keys_iterator]() { return keys_iterator->next(); }); } } -bool StorageS3::isColumnOriented() const +bool StorageS3::supportsSubsetOfColumns() const { - return FormatFactory::instance().checkIfFormatIsColumnOriented(format_name); + return FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(format_name); } Pipe StorageS3::read( const Names & column_names, const StorageSnapshotPtr & storage_snapshot, - SelectQueryInfo & /*query_info*/, + SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, unsigned num_streams) { + bool has_wildcards = s3_configuration.uri.bucket.find(PARTITION_ID_WILDCARD) != String::npos + || keys.back().find(PARTITION_ID_WILDCARD) != String::npos; + if (partition_by && has_wildcards) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Reading from a partitioned S3 storage is not implemented yet"); + updateS3Configuration(local_context, s3_configuration); Pipes pipes; @@ -702,11 +835,19 @@ Pipe StorageS3::read( requested_virtual_columns.push_back(virtual_column); } - std::shared_ptr iterator_wrapper = createFileIterator(s3_configuration, keys, is_key_with_globs, distributed_processing, local_context, read_tasks_used_in_schema_inference); + std::shared_ptr iterator_wrapper = createFileIterator( + s3_configuration, + keys, + is_key_with_globs, + distributed_processing, + local_context, + query_info.query, + virtual_block, + read_tasks_used_in_schema_inference); ColumnsDescription columns_description; Block block_for_format; - if (isColumnOriented()) + if (supportsSubsetOfColumns()) { auto fetch_columns = column_names; const auto & virtuals = getVirtuals(); @@ -999,7 +1140,8 @@ ColumnsDescription StorageS3::getTableStructureFromDataImpl( ContextPtr ctx, std::vector * read_keys_in_distributed_processing) { - auto file_iterator = createFileIterator(s3_configuration, {s3_configuration.uri.key}, is_key_with_globs, distributed_processing, ctx); + auto file_iterator + = createFileIterator(s3_configuration, {s3_configuration.uri.key}, is_key_with_globs, distributed_processing, ctx, nullptr, {}); ReadBufferIterator read_buffer_iterator = [&, first = false]() mutable -> std::unique_ptr { diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 3fde17682d0..ef16982ba58 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -34,25 +34,28 @@ class StorageS3Source : public SourceWithProgress, WithContext public: class DisclosedGlobIterator { - public: - DisclosedGlobIterator(const Aws::S3::S3Client &, const S3::URI &); - String next(); - private: - class Impl; - /// shared_ptr to have copy constructor - std::shared_ptr pimpl; + public: + DisclosedGlobIterator( + const Aws::S3::S3Client & client_, const S3::URI & globbed_uri_, ASTPtr query, const Block & virtual_header, ContextPtr context); + String next(); + + private: + class Impl; + /// shared_ptr to have copy constructor + std::shared_ptr pimpl; }; class KeysIterator { - public: - explicit KeysIterator(const std::vector & keys_); - String next(); + public: + explicit KeysIterator( + const std::vector & keys_, const String & bucket_, ASTPtr query, const Block & virtual_header, ContextPtr context); + String next(); - private: - class Impl; - /// shared_ptr to have copy constructor - std::shared_ptr pimpl; + private: + class Impl; + /// shared_ptr to have copy constructor + std::shared_ptr pimpl; }; class ReadTasksIterator @@ -203,6 +206,7 @@ private: S3Configuration s3_configuration; std::vector keys; NamesAndTypesList virtual_columns; + Block virtual_block; String format_name; String compression_method; @@ -222,6 +226,8 @@ private: bool is_key_with_globs, bool distributed_processing, ContextPtr local_context, + ASTPtr query, + const Block & virtual_block, const std::vector & read_tasks = {}); static ColumnsDescription getTableStructureFromDataImpl( @@ -234,7 +240,7 @@ private: ContextPtr ctx, std::vector * read_keys_in_distributed_processing = nullptr); - bool isColumnOriented() const override; + bool supportsSubsetOfColumns() const override; }; } diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index c0631311d8c..13be199bd37 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -82,6 +83,15 @@ StorageS3Cluster::StorageS3Cluster( storage_metadata.setConstraints(constraints_); setInMemoryMetadata(storage_metadata); + + auto default_virtuals = NamesAndTypesList{ + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; + + auto columns = storage_metadata.getSampleBlock().getNamesAndTypesList(); + virtual_columns = getVirtualsForStorage(columns, default_virtuals); + for (const auto & column : virtual_columns) + virtual_block.insert({column.type->createColumn(), column.type, column.name}); } /// The code executes on initiator @@ -98,11 +108,9 @@ Pipe StorageS3Cluster::read( auto cluster = context->getCluster(cluster_name)->getClusterWithReplicasAsShards(context->getSettingsRef()); - auto iterator = std::make_shared(*s3_configuration.client, s3_configuration.uri); - auto callback = std::make_shared([iterator]() mutable -> String - { - return iterator->next(); - }); + auto iterator = std::make_shared( + *s3_configuration.client, s3_configuration.uri, query_info.query, virtual_block, context); + auto callback = std::make_shared([iterator]() mutable -> String { return iterator->next(); }); /// Calculate the header. This is significant, because some columns could be thrown away in some cases like query with count(*) Block header = @@ -164,9 +172,7 @@ QueryProcessingStage::Enum StorageS3Cluster::getQueryProcessingStage( NamesAndTypesList StorageS3Cluster::getVirtuals() const { - return NamesAndTypesList{ - {"_path", std::make_shared(std::make_shared())}, - {"_file", std::make_shared(std::make_shared())}}; + return virtual_columns; } diff --git a/src/Storages/StorageS3Cluster.h b/src/Storages/StorageS3Cluster.h index 5470dda3e97..f823d1fdf04 100644 --- a/src/Storages/StorageS3Cluster.h +++ b/src/Storages/StorageS3Cluster.h @@ -49,6 +49,8 @@ private: String cluster_name; String format_name; String compression_method; + NamesAndTypesList virtual_columns; + Block virtual_block; }; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 0db4fa75aba..14931a6b230 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -582,9 +582,9 @@ ColumnsDescription IStorageURLBase::getTableStructureFromData( return readSchemaFromFormat(format, format_settings, read_buffer_iterator, urls_to_check.size() > 1, context); } -bool IStorageURLBase::isColumnOriented() const +bool IStorageURLBase::supportsSubsetOfColumns() const { - return FormatFactory::instance().checkIfFormatIsColumnOriented(format_name); + return FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(format_name); } Pipe IStorageURLBase::read( @@ -600,7 +600,7 @@ Pipe IStorageURLBase::read( ColumnsDescription columns_description; Block block_for_format; - if (isColumnOriented()) + if (supportsSubsetOfColumns()) { columns_description = storage_snapshot->getDescriptionForColumns(column_names); block_for_format = storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); @@ -687,7 +687,7 @@ Pipe StorageURLWithFailover::read( { ColumnsDescription columns_description; Block block_for_format; - if (isColumnOriented()) + if (supportsSubsetOfColumns()) { columns_description = storage_snapshot->getDescriptionForColumns(column_names); block_for_format = storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 25b88a827b6..85c77b00550 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -93,7 +93,7 @@ protected: QueryProcessingStage::Enum & processed_stage, size_t max_block_size) const; - bool isColumnOriented() const override; + bool supportsSubsetOfColumns() const override; private: virtual Block getHeaderBlock(const Names & column_names, const StorageSnapshotPtr & storage_snapshot) const = 0; diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index 2c2f1ec3034..f44daf2557e 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -140,7 +140,7 @@ SinkToStoragePtr StorageXDBC::write(const ASTPtr & /* query */, const StorageMet chooseCompressionMethod(uri, compression_method)); } -bool StorageXDBC::isColumnOriented() const +bool StorageXDBC::supportsSubsetOfColumns() const { return true; } diff --git a/src/Storages/StorageXDBC.h b/src/Storages/StorageXDBC.h index 910ba162f86..442db5277e0 100644 --- a/src/Storages/StorageXDBC.h +++ b/src/Storages/StorageXDBC.h @@ -67,7 +67,7 @@ private: Block getHeaderBlock(const Names & column_names, const StorageSnapshotPtr & storage_snapshot) const override; - bool isColumnOriented() const override; + bool supportsSubsetOfColumns() const override; }; } diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index 6674de06c07..a8edb8dd78b 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -85,6 +85,7 @@ StorageSystemParts::StorageSystemParts(const StorageID & table_id_) {"visible", std::make_shared()}, {"creation_tid", getTransactionIDDataType()}, + {"removal_tid_lock", std::make_shared()}, {"removal_tid", getTransactionIDDataType()}, {"creation_csn", std::make_shared()}, {"removal_csn", std::make_shared()}, @@ -295,6 +296,8 @@ void StorageSystemParts::processNextStorage( if (columns_mask[src_index++]) columns[res_index++]->insert(get_tid_as_field(part->version.creation_tid)); + if (columns_mask[src_index++]) + columns[res_index++]->insert(part->version.removal_tid_lock.load(std::memory_order_relaxed)); if (columns_mask[src_index++]) columns[res_index++]->insert(get_tid_as_field(part->version.getRemovalTID())); if (columns_mask[src_index++]) diff --git a/src/Storages/System/StorageSystemTransactions.cpp b/src/Storages/System/StorageSystemTransactions.cpp index 396fc875f74..21fa72ea12a 100644 --- a/src/Storages/System/StorageSystemTransactions.cpp +++ b/src/Storages/System/StorageSystemTransactions.cpp @@ -15,6 +15,7 @@ static DataTypePtr getStateEnumType() DataTypeEnum8::Values { {"RUNNING", static_cast(MergeTreeTransaction::State::RUNNING)}, + {"COMMITTING", static_cast(MergeTreeTransaction::State::COMMITTING)}, {"COMMITTED", static_cast(MergeTreeTransaction::State::COMMITTED)}, {"ROLLED_BACK", static_cast(MergeTreeTransaction::State::ROLLED_BACK)}, }); diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 57227ef307e..c1d1c1df1f1 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -404,7 +404,11 @@ def main(): elif args.image_path: pr_info.changed_files = set(i for i in args.image_path) else: - pr_info.fetch_changed_files() + try: + pr_info.fetch_changed_files() + except TypeError: + # If the event does not contain diff, nothing will be built + pass changed_images = get_changed_docker_images(pr_info, images_dict) if changed_images: diff --git a/tests/ci/docs_release.py b/tests/ci/docs_release.py index d404e79c312..806db28c1b1 100644 --- a/tests/ci/docs_release.py +++ b/tests/ci/docs_release.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import argparse import logging import subprocess import os @@ -15,17 +16,31 @@ from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version from commit_status_helper import get_commit from rerun_helper import RerunHelper +from tee_popen import TeePopen NAME = "Docs Release (actions)" + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="ClickHouse building script using prebuilt Docker image", + ) + parser.add_argument( + "--as-root", action="store_true", help="if the container should run as root" + ) + return parser.parse_args() + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) + args = parse_args() temp_path = TEMP_PATH repo_path = REPO_COPY gh = Github(get_best_robot_token()) - pr_info = PRInfo(need_changed_files=True) + pr_info = PRInfo() rerun_helper = RerunHelper(gh, pr_info, NAME) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") @@ -40,20 +55,23 @@ if __name__ == "__main__": if not os.path.exists(test_output): os.makedirs(test_output) - token = CLOUDFLARE_TOKEN - cmd = ( - "docker run --cap-add=SYS_PTRACE --volume=$SSH_AUTH_SOCK:/ssh-agent " - f"-e SSH_AUTH_SOCK=/ssh-agent -e CLOUDFLARE_TOKEN={token} " - f"-e EXTRA_BUILD_ARGS='--verbose' --volume={repo_path}:/repo_path" - f" --volume={test_output}:/output_path {docker_image}" - ) + if args.as_root: + user = "0:0" + else: + user = f"{os.geteuid()}:{os.getegid()}" run_log_path = os.path.join(test_output, "runlog.log") - with open(run_log_path, "w", encoding="utf-8") as log, SSHKey( - "ROBOT_CLICKHOUSE_SSH_KEY" - ): - with subprocess.Popen(cmd, shell=True, stderr=log, stdout=log) as process: + with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): + cmd = ( + f"docker run --cap-add=SYS_PTRACE --user={user} " + f"--volume='{os.getenv('SSH_AUTH_SOCK', '')}:/ssh-agent' " + f"--volume={repo_path}:/repo_path --volume={test_output}:/output_path " + f"-e SSH_AUTH_SOCK=/ssh-agent -e EXTRA_BUILD_ARGS='--verbose' " + f"-e CLOUDFLARE_TOKEN={CLOUDFLARE_TOKEN} {docker_image}" + ) + logging.info("Running command: %s", cmd) + with TeePopen(cmd, run_log_path) as process: retcode = process.wait() if retcode == 0: logging.info("Run successfully") @@ -98,3 +116,6 @@ if __name__ == "__main__": commit.create_status( context=NAME, description=description, state=status, target_url=report_url ) + + if status == "failure": + sys.exit(1) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 9d287d2a07e..d17e0f1f379 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -186,6 +186,7 @@ class PRInfo: else: self.diff_url = pull_request["diff_url"] else: + print("event.json does not match pull_request or push:") print(json.dumps(github_event, sort_keys=True, indent=4)) self.sha = os.getenv("GITHUB_SHA") self.number = 0 @@ -204,8 +205,8 @@ class PRInfo: self.fetch_changed_files() def fetch_changed_files(self): - if not self.diff_url: - raise Exception("Diff URL cannot be find for event") + if not getattr(self, "diff_url", False): + raise TypeError("The event does not have diff URL") response = get_with_retries( self.diff_url, diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index bd70134760a..87139c5bb8a 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -253,16 +253,7 @@ if __name__ == "__main__": ) sys.exit(1) else: - if "pr-documentation" in pr_info.labels or "pr-doc-fix" in pr_info.labels: - commit.create_status( - context=NAME, - description="Skipping checks for documentation", - state="success", - target_url=url, - ) - print("::notice ::Can run, but it's documentation PR, skipping") - else: - print("::notice ::Can run") - commit.create_status( - context=NAME, description=description, state="pending", target_url=url - ) + print("::notice ::Can run") + commit.create_status( + context=NAME, description=description, state="pending", target_url=url + ) diff --git a/tests/config/config.d/transactions.xml b/tests/config/config.d/transactions.xml index 19810986ea1..9948b1f1865 100644 --- a/tests/config/config.d/transactions.xml +++ b/tests/config/config.d/transactions.xml @@ -10,4 +10,12 @@ 7500 + + /test/clickhouse/txn + + 0.0 + + + 0.01 + diff --git a/tests/integration/test_keeper_force_recovery_single_node/__init__.py b/tests/integration/test_keeper_force_recovery_single_node/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_keeper_force_recovery_single_node/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper1.xml b/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper1.xml new file mode 100644 index 00000000000..441c1bc185d --- /dev/null +++ b/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper1.xml @@ -0,0 +1,33 @@ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 75 + trace + + + + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper1_solo.xml b/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper1_solo.xml new file mode 100644 index 00000000000..f0cb887b062 --- /dev/null +++ b/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper1_solo.xml @@ -0,0 +1,24 @@ + + + 1 + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 75 + trace + + + + + 1 + node1 + 9234 + + + + diff --git a/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper2.xml b/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper2.xml new file mode 100644 index 00000000000..e2e2c1fd7db --- /dev/null +++ b/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper2.xml @@ -0,0 +1,33 @@ + + + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 75 + trace + + + + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper3.xml b/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper3.xml new file mode 100644 index 00000000000..e2ac0400d88 --- /dev/null +++ b/tests/integration/test_keeper_force_recovery_single_node/configs/enable_keeper3.xml @@ -0,0 +1,33 @@ + + + 9181 + 3 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 75 + trace + + + + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_keeper_force_recovery_single_node/configs/use_keeper.xml b/tests/integration/test_keeper_force_recovery_single_node/configs/use_keeper.xml new file mode 100644 index 00000000000..384e984f210 --- /dev/null +++ b/tests/integration/test_keeper_force_recovery_single_node/configs/use_keeper.xml @@ -0,0 +1,16 @@ + + + + node1 + 9181 + + + node2 + 9181 + + + node3 + 9181 + + + diff --git a/tests/integration/test_keeper_force_recovery_single_node/test.py b/tests/integration/test_keeper_force_recovery_single_node/test.py new file mode 100644 index 00000000000..1e58a25221e --- /dev/null +++ b/tests/integration/test_keeper_force_recovery_single_node/test.py @@ -0,0 +1,157 @@ +import os +import pytest +import socket +from helpers.cluster import ClickHouseCluster +import time + + +from kazoo.client import KazooClient + +CLUSTER_SIZE = 3 + +cluster = ClickHouseCluster(__file__) +CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") + + +def get_nodes(): + nodes = [] + for i in range(CLUSTER_SIZE): + nodes.append( + cluster.add_instance( + f"node{i+1}", + main_configs=[ + f"configs/enable_keeper{i+1}.xml", + f"configs/use_keeper.xml", + ], + stay_alive=True, + ) + ) + + return nodes + + +nodes = get_nodes() + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + finally: + cluster.shutdown() + + +def get_fake_zk(nodename, timeout=30.0): + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) + _fake_zk_instance.start() + return _fake_zk_instance + + +def get_keeper_socket(node_name): + hosts = cluster.get_instance_ip(node_name) + client = socket.socket() + client.settimeout(10) + client.connect((hosts, 9181)) + return client + + +def send_4lw_cmd(node_name, cmd="ruok"): + client = None + try: + client = get_keeper_socket(node_name) + client.send(cmd.encode()) + data = client.recv(100_000) + data = data.decode() + return data + finally: + if client is not None: + client.close() + + +def wait_until_connected(node_name): + while send_4lw_cmd(node_name, "mntr") == NOT_SERVING_REQUESTS_ERROR_MSG: + time.sleep(0.1) + + +def wait_nodes(nodes): + for node in nodes: + wait_until_connected(node.name) + + +def wait_and_assert_data(zk, path, data): + while zk.exists(path) is None: + time.sleep(0.1) + assert zk.get(path)[0] == data.encode() + + +def close_zk(zk): + zk.stop() + zk.close() + + +NOT_SERVING_REQUESTS_ERROR_MSG = "This instance is not currently serving requests" + + +def test_cluster_recovery(started_cluster): + node_zks = [] + try: + wait_nodes(nodes) + + node_zks = [get_fake_zk(node.name) for node in nodes] + + data_in_cluster = [] + + def add_data(zk, path, data): + zk.create(path, data.encode()) + data_in_cluster.append((path, data)) + + def assert_all_data(zk): + for path, data in data_in_cluster: + wait_and_assert_data(zk, path, data) + + for i, zk in enumerate(node_zks): + add_data(zk, f"/test_force_recovery_node{i+1}", f"somedata{i+1}") + + for zk in node_zks: + assert_all_data(zk) + + nodes[0].stop_clickhouse() + + add_data(node_zks[1], "/test_force_recovery_extra", "somedataextra") + + for node_zk in node_zks[2:CLUSTER_SIZE]: + wait_and_assert_data(node_zk, "/test_force_recovery_extra", "somedataextra") + + nodes[0].start_clickhouse() + wait_until_connected(nodes[0].name) + + node_zks[0] = get_fake_zk(nodes[0].name) + wait_and_assert_data(node_zks[0], "/test_force_recovery_extra", "somedataextra") + + # stop all nodes + for node_zk in node_zks: + close_zk(node_zk) + node_zks = [] + + for node in nodes: + node.stop_clickhouse() + + nodes[0].copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper1_solo.xml"), + "/etc/clickhouse-server/config.d/enable_keeper1.xml", + ) + + nodes[0].start_clickhouse() + wait_until_connected(nodes[0].name) + + assert_all_data(get_fake_zk(nodes[0].name)) + finally: + try: + for zk_conn in node_zks: + close_zk(zk_conn) + except: + pass diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/__init__.py b/tests/integration/test_replicated_merge_tree_s3_restore/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml new file mode 100644 index 00000000000..31f7e06db3c --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml @@ -0,0 +1,40 @@ + + + + + + true + + node1z + 9000 + + + node2z + 9000 + + + + + + true + + node1n + 9000 + + + node2n + 9000 + + + + + + true + + node_another_bucket + 9000 + + + + + diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml new file mode 100644 index 00000000000..63fe7ac1769 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml @@ -0,0 +1,35 @@ + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + true + 1 + + + local + / + + + + + +
+ s3 +
+ + hdd + + + + + + + + 0 + 1 + + diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml new file mode 100644 index 00000000000..920db2c9edd --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml @@ -0,0 +1,34 @@ + + + + + s3 + http://minio1:9001/root2/data/ + minio + minio123 + true + 1 + + + local + / + + + + + +
+ s3 +
+ + hdd + + + + + + + + 0 + + diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml new file mode 100644 index 00000000000..9194d779d16 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml @@ -0,0 +1,35 @@ + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + true + 1 + + + local + / + + + + + +
+ s3 +
+ + hdd + + + + + + + + 0 + 0 + + diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/test.py b/tests/integration/test_replicated_merge_tree_s3_restore/test.py new file mode 100644 index 00000000000..904bcfa4280 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_restore/test.py @@ -0,0 +1,258 @@ +import os +import logging +import random +import string +import time + +import pytest +from helpers.cluster import ClickHouseCluster, get_instances_dir + + +COMMON_CONFIGS = ["configs/config.d/clusters.xml"] + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + + cluster.add_instance( + "node1z", + main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf.xml"], + macros={"cluster": "node_zero_copy", "replica": "0"}, + with_minio=True, + with_zookeeper=True, + stay_alive=True, + ) + cluster.add_instance( + "node2z", + main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf.xml"], + macros={"cluster": "node_zero_copy", "replica": "1"}, + with_zookeeper=True, + stay_alive=True, + ) + cluster.add_instance( + "node1n", + main_configs=COMMON_CONFIGS + + ["configs/config.d/storage_conf_without_zero_copy.xml"], + macros={"cluster": "node_no_zero_copy", "replica": "2"}, + with_minio=True, + with_zookeeper=True, + stay_alive=True, + ) + cluster.add_instance( + "node2n", + main_configs=COMMON_CONFIGS + + ["configs/config.d/storage_conf_without_zero_copy.xml"], + macros={"cluster": "node_no_zero_copy", "replica": "3"}, + with_zookeeper=True, + stay_alive=True, + ) + cluster.add_instance( + "node_another_bucket", + main_configs=COMMON_CONFIGS + + ["configs/config.d/storage_conf_another_bucket.xml"], + macros={"cluster": "node_another_bucket", "replica": "0"}, + with_zookeeper=True, + stay_alive=True, + ) + + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + yield cluster + finally: + cluster.shutdown() + + +def random_string(length): + letters = string.ascii_letters + return "".join(random.choice(letters) for i in range(length)) + + +def create_table(node, table_name, schema, attach=False, db_atomic=False, uuid=""): + node.query( + "CREATE DATABASE IF NOT EXISTS s3 {on_cluster} ENGINE = {engine}".format( + engine="Atomic" if db_atomic else "Ordinary", + on_cluster="ON CLUSTER '{cluster}'", + ) + ) + + create_table_statement = """ + {create} TABLE s3.{table_name} {uuid} {on_cluster} ( + key UInt32, + {schema} + ) ENGINE={engine} + PARTITION BY key + ORDER BY key + SETTINGS + storage_policy='s3', + old_parts_lifetime=600, + index_granularity=512 + """.format( + create="ATTACH" if attach else "CREATE", + table_name=table_name, + uuid="UUID '{uuid}'".format(uuid=uuid) if db_atomic and uuid else "", + on_cluster="ON CLUSTER '{cluster}'", + schema=schema, + engine="ReplicatedMergeTree('/clickhouse/tables/{cluster}/test', '{replica}')", + ) + + node.query(create_table_statement) + + +def purge_s3(cluster, bucket): + minio = cluster.minio_client + for obj in list(minio.list_objects(bucket, recursive=True)): + if str(obj.object_name).find(".SCHEMA_VERSION") != -1: + continue + minio.remove_object(bucket, obj.object_name) + + +def drop_s3_metadata(node): + node.exec_in_container( + ["bash", "-c", "rm -rf /var/lib/clickhouse/disks/s3/*"], user="root" + ) + + +def drop_shadow_information(node): + node.exec_in_container( + ["bash", "-c", "rm -rf /var/lib/clickhouse/shadow/*"], user="root" + ) + + +def create_restore_file(node, revision=None, bucket=None, path=None, detached=None): + node.exec_in_container( + ["bash", "-c", "mkdir -p /var/lib/clickhouse/disks/s3/"], user="root" + ) + node.exec_in_container( + ["bash", "-c", "touch /var/lib/clickhouse/disks/s3/restore"], user="root" + ) + + add_restore_option = 'echo -en "{}={}\n" >> /var/lib/clickhouse/disks/s3/restore' + if revision: + node.exec_in_container( + ["bash", "-c", add_restore_option.format("revision", revision)], user="root" + ) + if bucket: + node.exec_in_container( + ["bash", "-c", add_restore_option.format("source_bucket", bucket)], + user="root", + ) + if path: + node.exec_in_container( + ["bash", "-c", add_restore_option.format("source_path", path)], user="root" + ) + if detached: + node.exec_in_container( + ["bash", "-c", add_restore_option.format("detached", "true")], user="root" + ) + + +def get_revision_counter(node, backup_number): + return int( + node.exec_in_container( + [ + "bash", + "-c", + "cat /var/lib/clickhouse/disks/s3/shadow/{}/revision.txt".format( + backup_number + ), + ], + user="root", + ) + ) + + +def get_table_uuid(node, db_atomic, table): + uuid = "" + if db_atomic: + uuid = node.query( + "SELECT uuid FROM system.tables WHERE database='s3' AND table='{}' FORMAT TabSeparated".format( + table + ) + ).strip() + return uuid + + +@pytest.fixture(autouse=True) +def drop_table(cluster): + yield + + node_names = ["node1z", "node2z", "node1n", "node2n", "node_another_bucket"] + + for node_name in node_names: + node = cluster.instances[node_name] + node.query("DROP TABLE IF EXISTS s3.test SYNC") + node.query("DROP DATABASE IF EXISTS s3 SYNC") + + drop_s3_metadata(node) + drop_shadow_information(node) + + buckets = [cluster.minio_bucket, cluster.minio_bucket_2] + for bucket in buckets: + purge_s3(cluster, bucket) + + +@pytest.mark.parametrize("db_atomic", [False, True]) +@pytest.mark.parametrize("zero_copy", [False, True]) +def test_restore_another_bucket_path(cluster, db_atomic, zero_copy): + suffix = "z" if zero_copy else "n" + nodes = [cluster.instances[f"node1{suffix}"], cluster.instances[f"node2{suffix}"]] + + keys = 100 + data_columns = 10 + size = 1 + + columns = [] + for c in range(0, data_columns): + columns.append("data{c} String".format(c=c)) + schema = ", ".join(columns) + + create_table(nodes[0], "test", schema, db_atomic=db_atomic) + uuid = get_table_uuid(nodes[0], db_atomic, "test") + + dropped_keys = 0 + + for key in range(0, keys): + node = nodes[key % 2] + node.query( + "INSERT INTO s3.test SELECT {key}, * FROM generateRandom('{schema}') LIMIT {size}".format( + key=key, schema=schema, size=size + ) + ) + if not (key % 3): + dropped_keys += 1 + node.query("ALTER TABLE s3.test DROP PARTITION '{key}'".format(key=key)) + + for key in range(0, keys): + if not ((key + 1) % 3): + dropped_keys += 1 + node.query("ALTER TABLE s3.test DROP PARTITION '{key}'".format(key=key)) + + nodes[0].query("SYSTEM SYNC REPLICA s3.test") + nodes[1].query("SYSTEM SYNC REPLICA s3.test") + + # To ensure parts have merged + nodes[0].query("OPTIMIZE TABLE s3.test") + nodes[1].query("OPTIMIZE TABLE s3.test") + + assert nodes[0].query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(size * (keys - dropped_keys)) + assert nodes[1].query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(size * (keys - dropped_keys)) + + node_another_bucket = cluster.instances["node_another_bucket"] + + create_restore_file(node_another_bucket, bucket="root") + node_another_bucket.query("SYSTEM RESTART DISK s3") + create_table( + node_another_bucket, "test", schema, attach=True, db_atomic=db_atomic, uuid=uuid + ) + + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(size * (keys - dropped_keys)) diff --git a/tests/performance/formats_columns_sampling.xml b/tests/performance/formats_columns_sampling.xml new file mode 100644 index 00000000000..25f9dc000a3 --- /dev/null +++ b/tests/performance/formats_columns_sampling.xml @@ -0,0 +1,32 @@ + + + 1 + + + + + format + + TabSeparatedWithNames + CustomSeparatedWithNames + CSVWithNames + JSONEachRow + JSONCompactEachRowWithNames + TSKV + Avro + ORC + Parquet + Arrow + Native + + + + + CREATE TABLE IF NOT EXISTS table_{format} ENGINE = File({format}) AS test.hits + + INSERT INTO table_{format} SELECT * FROM test.hits LIMIT 100000 + + SELECT WatchID FROM table_{format} FORMAT Null + + DROP TABLE IF EXISTS table_{format} + diff --git a/tests/performance/norm_distance.xml b/tests/performance/norm_distance.xml new file mode 100644 index 00000000000..b6a7f9724c2 --- /dev/null +++ b/tests/performance/norm_distance.xml @@ -0,0 +1,100 @@ + + + + + element_type + + UInt8 + Int16 + Int32 + Int64 + Float32 + Float64 + + + + + + CREATE TABLE vecs_{element_type} ( + v Array({element_type}) + ) ENGINE=Memory; + + + + + + INSERT INTO vecs_{element_type} + SELECT v FROM ( + SELECT + number AS n, + [ + rand(n*10), + rand(n*10+1), + rand(n*10+2), + rand(n*10+3), + rand(n*10+4), + rand(n*10+5), + rand(n*10+6), + rand(n*10+7), + rand(n*10+8), + rand(n*10+9) + ] AS v + FROM system.numbers + LIMIT 10000000 + ); + + + + + + CREATE TABLE tuples_{element_type} ( + t Tuple( + {element_type}, + {element_type}, + {element_type}, + {element_type}, + {element_type}, + {element_type}, + {element_type}, + {element_type}, + {element_type}, + {element_type} + ) + ) ENGINE=Memory; + + + + INSERT INTO tuples_{element_type} + SELECT (v[1], v[2], v[3], v[4], v[5], v[6], v[7], v[8], v[9], v[10]) FROM vecs_{element_type}; + + + + 1 + + + + + + norm + + L1 + L2 + Linf + + + + + + SELECT sum(dist) FROM (SELECT {norm}Norm(t) AS dist FROM tuples_{element_type}) + WITH (SELECT t FROM tuples_{element_type} limit 1) AS a SELECT sum(dist) FROM (SELECT {norm}Distance(a, t) AS dist FROM tuples_{element_type}) + WITH (SELECT t FROM tuples_{element_type} limit 1) AS a SELECT sum(dist) FROM (SELECT cosineDistance(a, t) AS dist FROM tuples_{element_type}) + + + SELECT sum(dist) FROM (SELECT array{norm}Norm(v) AS dist FROM vecs_{element_type}) + WITH (SELECT v FROM vecs_{element_type} limit 1) AS a SELECT sum(dist) FROM (SELECT array{norm}Distance(a, v) AS dist FROM vecs_{element_type}) + WITH (SELECT v FROM vecs_{element_type} limit 1) AS a SELECT sum(dist) FROM (SELECT arrayCosineDistance(a, v) AS dist FROM vecs_{element_type}) + + DROP TABLE vecs_{element_type} + DROP TABLE tuples_{element_type} + + diff --git a/tests/performance/unary_arithmetic_functions.xml b/tests/performance/unary_arithmetic_functions.xml index 62e11457ac4..93dd5244c9b 100644 --- a/tests/performance/unary_arithmetic_functions.xml +++ b/tests/performance/unary_arithmetic_functions.xml @@ -1,6 +1,4 @@ - - func diff --git a/tests/queries/0_stateless/00948_values_interpreter_template.sql b/tests/queries/0_stateless/00948_values_interpreter_template.sql index 2404c99f736..a3d2ffd7452 100644 --- a/tests/queries/0_stateless/00948_values_interpreter_template.sql +++ b/tests/queries/0_stateless/00948_values_interpreter_template.sql @@ -21,7 +21,7 @@ INSERT INTO type_names VALUES (1, toTypeName([1, 2]), toTypeName((256, -1, 3.14, -- _NUM_5: Float64 -> Int64 INSERT INTO values_template VALUES ((1), lower(replaceAll('Hella', 'a', 'o')), 1 + 2 + 3, round(-4 * 5.0), nan / CAST('42', 'Int8'), reverse([1, 2, 3])), ((2), lower(replaceAll('Warld', 'a', 'o')), -4 + 5 + 6, round(18446744073709551615 * 1e-19), 1.0 / CAST('0', 'Int8'), reverse([])), ((3), lower(replaceAll('Test', 'a', 'o')), 3 + 2 + 1, round(9223372036854775807 * -1), 6.28 / CAST('2', 'Int8'), reverse([4, 5])), ((4), lower(replaceAll('Expressians', 'a', 'o')), 6 + 5 + 4, round(1 * -9223372036854775807), 127.0 / CAST('127', 'Int8'), reverse([6, 7, 8, 9, 0])); -INSERT INTO values_template_nullable VALUES ((1), lower(replaceAll('Hella', 'a', 'o')), 1 + 2 + 3, arraySort(x -> assumeNotNull(x), [null, NULL])), ((2), lower(replaceAll('Warld', 'b', 'o')), 4 - 5 + 6, arraySort(x -> assumeNotNull(x), [+1, -1, Null])), ((3), lower(replaceAll('Test', 'c', 'o')), 3 + 2 - 1, arraySort(x -> assumeNotNull(x), [1, nUlL, 3.14])), ((4), lower(replaceAll(null, 'c', 'o')), 6 + 5 - null, arraySort(x -> assumeNotNull(x), [3, 2, 1])); +INSERT INTO values_template_nullable VALUES ((1), lower(replaceAll('Hella', 'a', 'o')), 1 + 2 + 3, arraySort(x -> assumeNotNull(x), [null, NULL::Nullable(UInt8)])), ((2), lower(replaceAll('Warld', 'b', 'o')), 4 - 5 + 6, arraySort(x -> assumeNotNull(x), [+1, -1, Null])), ((3), lower(replaceAll('Test', 'c', 'o')), 3 + 2 - 1, arraySort(x -> assumeNotNull(x), [1, nUlL, 3.14])), ((4), lower(replaceAll(null, 'c', 'o')), 6 + 5 - null, arraySort(x -> assumeNotNull(x), [3, 2, 1])); INSERT INTO values_template_fallback VALUES (1 + x); -- { clientError 62 } INSERT INTO values_template_fallback VALUES (abs(functionThatDoesNotExists(42))); -- { clientError 46 } diff --git a/tests/queries/0_stateless/01133_begin_commit_race.reference b/tests/queries/0_stateless/01133_begin_commit_race.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01133_begin_commit_race.sh b/tests/queries/0_stateless/01133_begin_commit_race.sh new file mode 100755 index 00000000000..f64570950c7 --- /dev/null +++ b/tests/queries/0_stateless/01133_begin_commit_race.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Tags: long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -e + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS mt"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE mt (n Int64) ENGINE=MergeTree ORDER BY n SETTINGS old_parts_lifetime=0"; + + +function begin_commit_readonly() +{ + $CLICKHOUSE_CLIENT --multiquery --query " + SET wait_changes_become_visible_after_commit_mode='wait'; + BEGIN TRANSACTION; + COMMIT;" 2>&1| grep -Fa "Exception: " | grep -Fv UNKNOWN_STATUS_OF_TRANSACTION +} + +function begin_rollback_readonly() +{ + $CLICKHOUSE_CLIENT --wait_changes_become_visible_after_commit_mode=wait_unknown --multiquery --query " + BEGIN TRANSACTION; + SET TRANSACTION SNAPSHOT 42; + ROLLBACK;" +} + +function begin_insert_commit() +{ + $CLICKHOUSE_CLIENT --wait_changes_become_visible_after_commit_mode=async --multiquery --query " + BEGIN TRANSACTION; + INSERT INTO mt VALUES ($RANDOM); + COMMIT;" 2>&1| grep -Fa "Exception: " | grep -Fv UNKNOWN_STATUS_OF_TRANSACTION +} + +function introspection() +{ + $CLICKHOUSE_CLIENT -q "SELECT * FROM system.transactions FORMAT Null" + $CLICKHOUSE_CLIENT -q "SELECT transactionLatestSnapshot(), transactionOldestSnapshot() FORMAT Null" +} + +export -f begin_commit_readonly +export -f begin_rollback_readonly +export -f begin_insert_commit +export -f introspection + +TIMEOUT=20 + +clickhouse_client_loop_timeout $TIMEOUT begin_commit_readonly & +clickhouse_client_loop_timeout $TIMEOUT begin_rollback_readonly & +clickhouse_client_loop_timeout $TIMEOUT begin_insert_commit & +clickhouse_client_loop_timeout $TIMEOUT introspection & + +wait + +$CLICKHOUSE_CLIENT --query "DROP TABLE mt"; diff --git a/tests/queries/0_stateless/01172_transaction_counters.sql b/tests/queries/0_stateless/01172_transaction_counters.sql index 5431673fd62..b84a7b25c47 100644 --- a/tests/queries/0_stateless/01172_transaction_counters.sql +++ b/tests/queries/0_stateless/01172_transaction_counters.sql @@ -42,7 +42,13 @@ rollback; system flush logs; select indexOf((select arraySort(groupUniqArray(tid)) from system.transactions_info_log where database=currentDatabase() and table='txn_counters'), tid), - (toDecimal64(now64(6), 6) - toDecimal64(event_time, 6)) < 100, type, thread_id!=0, length(query_id)=length(queryID()), tid_hash!=0, csn=0, part + (toDecimal64(now64(6), 6) - toDecimal64(event_time, 6)) < 100, + type, + thread_id!=0, + length(query_id)=length(queryID()) or type='Commit' and query_id='', -- ignore fault injection after commit + tid_hash!=0, + csn=0, + part from system.transactions_info_log where tid in (select tid from system.transactions_info_log where database=currentDatabase() and table='txn_counters' and not (tid.1=1 and tid.2=1)) or (database=currentDatabase() and table='txn_counters') order by event_time; diff --git a/tests/queries/0_stateless/01269_create_with_null.reference b/tests/queries/0_stateless/01269_create_with_null.reference index 73f834da75a..4e52c4a42d6 100644 --- a/tests/queries/0_stateless/01269_create_with_null.reference +++ b/tests/queries/0_stateless/01269_create_with_null.reference @@ -1,7 +1,7 @@ Nullable(Int32) Int32 Nullable(Int32) Int32 CREATE TABLE default.data_null\n(\n `a` Nullable(Int32),\n `b` Int32,\n `c` Nullable(Int32),\n `d` Int32\n)\nENGINE = Memory -Nullable(Int32) Int32 Nullable(Int32) Nullable(Int32) -CREATE TABLE default.set_null\n(\n `a` Nullable(Int32),\n `b` Int32,\n `c` Nullable(Int32),\n `d` Nullable(Int32)\n)\nENGINE = Memory -CREATE TABLE default.set_null\n(\n `a` Nullable(Int32),\n `b` Int32,\n `c` Nullable(Int32),\n `d` Nullable(Int32)\n)\nENGINE = Memory +Nullable(Int32) Int32 Nullable(Int32) Nullable(Int32) Nullable(UInt8) +CREATE TABLE default.set_null\n(\n `a` Nullable(Int32),\n `b` Int32,\n `c` Nullable(Int32),\n `d` Nullable(Int32),\n `f` Nullable(UInt8) DEFAULT 1\n)\nENGINE = Memory +CREATE TABLE default.set_null\n(\n `a` Nullable(Int32),\n `b` Int32,\n `c` Nullable(Int32),\n `d` Nullable(Int32),\n `f` Nullable(UInt8) DEFAULT 1\n)\nENGINE = Memory CREATE TABLE default.cannot_be_nullable\n(\n `n` Nullable(Int8),\n `a` Array(UInt8)\n)\nENGINE = Memory CREATE TABLE default.cannot_be_nullable\n(\n `n` Nullable(Int8),\n `a` Array(UInt8)\n)\nENGINE = Memory diff --git a/tests/queries/0_stateless/01269_create_with_null.sql b/tests/queries/0_stateless/01269_create_with_null.sql index 7548070ce4b..ac57f613dfd 100644 --- a/tests/queries/0_stateless/01269_create_with_null.sql +++ b/tests/queries/0_stateless/01269_create_with_null.sql @@ -39,13 +39,14 @@ CREATE TABLE set_null ( a INT NULL, b INT NOT NULL, c Nullable(INT), - d INT + d INT, + f DEFAULT 1 ) engine=Memory(); -INSERT INTO set_null VALUES (NULL, 2, NULL, NULL); +INSERT INTO set_null VALUES (NULL, 2, NULL, NULL, NULL); -SELECT toTypeName(a), toTypeName(b), toTypeName(c), toTypeName(d) FROM set_null; +SELECT toTypeName(a), toTypeName(b), toTypeName(c), toTypeName(d), toTypeName(f) FROM set_null; SHOW CREATE TABLE set_null; DETACH TABLE set_null; diff --git a/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql b/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql index 557bd297436..add38dbd3f8 100644 --- a/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql +++ b/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql @@ -55,5 +55,5 @@ FROM numbers(100000); SET allow_experimental_projection_optimization=1, optimize_aggregation_in_order=1, force_optimize_projection = 1; -WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM normal WHERE ts > '2021-12-06 22:00:00' GROUP BY a ORDER BY v LIMIT 5; -WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM normal WHERE ts > '2021-12-06 22:00:00' GROUP BY toStartOfHour(ts), a ORDER BY v LIMIT 5; +WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM agg WHERE ts > '2021-12-06 22:00:00' GROUP BY a ORDER BY v LIMIT 5; +WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM agg WHERE ts > '2021-12-06 22:00:00' GROUP BY toStartOfHour(ts), a ORDER BY v LIMIT 5; diff --git a/tests/queries/0_stateless/02000_join_on_const.reference b/tests/queries/0_stateless/02000_join_on_const.reference index b9494e4689c..3035351fd87 100644 --- a/tests/queries/0_stateless/02000_join_on_const.reference +++ b/tests/queries/0_stateless/02000_join_on_const.reference @@ -4,6 +4,7 @@ 1 1 1 +1 - ON NULL - - inner - - left - diff --git a/tests/queries/0_stateless/02000_join_on_const.sql b/tests/queries/0_stateless/02000_join_on_const.sql index f6d686cf9bc..92ded98b5f4 100644 --- a/tests/queries/0_stateless/02000_join_on_const.sql +++ b/tests/queries/0_stateless/02000_join_on_const.sql @@ -13,8 +13,8 @@ SELECT 70 = 10 * sum(t1.id) + sum(t2.id) AND count() == 4 FROM t1 JOIN t2 ON 2 = SELECT 70 = 10 * sum(t1.id) + sum(t2.id) AND count() == 4 FROM t1 INNER ANY JOIN t2 ON toNullable(1); SELECT 70 = 10 * sum(t1.id) + sum(t2.id) AND count() == 4 FROM t1 INNER ANY JOIN t2 ON toLowCardinality(1); SELECT 70 = 10 * sum(t1.id) + sum(t2.id) AND count() == 4 FROM t1 INNER ANY JOIN t2 ON toLowCardinality(toNullable(1)); +SELECT 70 = 10 * sum(t1.id) + sum(t2.id) AND count() == 4 FROM t1 INNER ANY JOIN t2 ON toNullable(toLowCardinality(1)); -SELECT * FROM t1 INNER ANY JOIN t2 ON toNullable(toLowCardinality(1)); -- { serverError 403 } SELECT * FROM t1 INNER ANY JOIN t2 ON toUInt16(1); -- { serverError 403 } SELECT * FROM t1 INNER ANY JOIN t2 ON toInt8(1); -- { serverError 403 } SELECT * FROM t1 INNER ANY JOIN t2 ON 256; -- { serverError 403 } diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index ad18e38adcc..d4ada9ba5c8 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -485,6 +485,7 @@ CREATE TABLE system.parts `projections` Array(String), `visible` UInt8, `creation_tid` Tuple(UInt64, UInt64, UUID), + `removal_tid_lock` UInt64, `removal_tid` Tuple(UInt64, UInt64, UUID), `creation_csn` UInt64, `removal_csn` UInt64, diff --git a/tests/queries/0_stateless/02242_negative_datetime64.reference b/tests/queries/0_stateless/02242_negative_datetime64.reference index 7f14679ac56..fbbebb520ae 100644 --- a/tests/queries/0_stateless/02242_negative_datetime64.reference +++ b/tests/queries/0_stateless/02242_negative_datetime64.reference @@ -1,2 +1,3 @@ -127914467.877 187618332.123 +1969-12-31 23:59:59.123 diff --git a/tests/queries/0_stateless/02242_negative_datetime64.sql b/tests/queries/0_stateless/02242_negative_datetime64.sql index 32086188608..40679841943 100644 --- a/tests/queries/0_stateless/02242_negative_datetime64.sql +++ b/tests/queries/0_stateless/02242_negative_datetime64.sql @@ -1,2 +1,3 @@ SELECT cast(toDateTime64('1965-12-12 12:12:12.123', 3, 'UTC') as Decimal64(3)); SELECT cast(toDateTime64('1975-12-12 12:12:12.123', 3, 'UTC') as Decimal64(3)); +SELECT toDateTime64('1969-12-31 23:59:59.123', 3, 'UTC'); diff --git a/tests/queries/0_stateless/02282_array_distance.reference b/tests/queries/0_stateless/02282_array_distance.reference index 158df656403..2fd6c66c817 100644 --- a/tests/queries/0_stateless/02282_array_distance.reference +++ b/tests/queries/0_stateless/02282_array_distance.reference @@ -1,7 +1,7 @@ 6 3.7416575 3 -0.0025851727 +0.002585097 \N nan 12 @@ -13,14 +13,14 @@ nan 2 5 4 -0.16847819 +0.16847816 0.35846698 -0.07417989 +0.0741799 6 8 9 0.020204102886728692 -0.11808289631180302 +0.11808289631180313 0 1 1 218.74642854227358 1 2 1348.2117786164013 diff --git a/tests/queries/0_stateless/02293_formats_json_columns.sh b/tests/queries/0_stateless/02293_formats_json_columns.sh index 20eba0449d8..74d9a4f5aab 100755 --- a/tests/queries/0_stateless/02293_formats_json_columns.sh +++ b/tests/queries/0_stateless/02293_formats_json_columns.sh @@ -50,7 +50,7 @@ echo ' $CLICKHOUSE_CLIENT -q "desc file(data_02293, JSONColumns)" $CLICKHOUSE_CLIENT -q "select * from file(data_02293, JSONColumns)" -$CLICKHOUSE_CLIENT -q "select * from file(data_02293, JSONColumns, 'a UInt32, t String')" 2>&1 | grep -F -q 'INCORRECT_DATA' && echo 'OK' || echo 'FAIL' +$CLICKHOUSE_CLIENT -q "select * from file(data_02293, JSONColumns, 'a UInt32, t String') settings input_format_skip_unknown_fields=0" 2>&1 | grep -F -q 'INCORRECT_DATA' && echo 'OK' || echo 'FAIL' $CLICKHOUSE_CLIENT -q "select * from file(data_02293, JSONColumns, 'a UInt32, t String') settings input_format_skip_unknown_fields=1" echo ' diff --git a/tests/queries/0_stateless/02293_http_header_full_summary_without_progress.reference b/tests/queries/0_stateless/02293_http_header_full_summary_without_progress.reference new file mode 100644 index 00000000000..538ac795107 --- /dev/null +++ b/tests/queries/0_stateless/02293_http_header_full_summary_without_progress.reference @@ -0,0 +1,2 @@ +Read rows in summary is not zero +< HTTP/1.1 408 Request Time-out diff --git a/tests/queries/0_stateless/02293_http_header_full_summary_without_progress.sh b/tests/queries/0_stateless/02293_http_header_full_summary_without_progress.sh new file mode 100755 index 00000000000..8f08bd6f84b --- /dev/null +++ b/tests/queries/0_stateless/02293_http_header_full_summary_without_progress.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +CURL_OUTPUT=$(echo 'SELECT 1 + sleepEachRow(0.00002) FROM numbers(100000)' | \ + ${CLICKHOUSE_CURL_COMMAND} -v "${CLICKHOUSE_URL}&wait_end_of_query=1&send_progress_in_http_headers=0&max_execution_time=1" --data-binary @- 2>&1) + +READ_ROWS=$(echo "${CURL_OUTPUT}" | \ + grep 'X-ClickHouse-Summary' | \ + awk '{print $3}' | \ + sed -E 's/.*"read_rows":"?([^,"]*)"?.*/\1/' + ) + +if [ "$READ_ROWS" -ne 0 ]; +then + echo "Read rows in summary is not zero" +else + echo "Read rows in summary is zero!" +fi + +# Check that the response code is correct too +echo "${CURL_OUTPUT}" | grep "< HTTP/1.1" diff --git a/tests/queries/0_stateless/02293_http_header_summary_contains_exception_code_with_progress.reference b/tests/queries/0_stateless/02293_http_header_summary_contains_exception_code_with_progress.reference new file mode 100644 index 00000000000..487bd5d5bc3 --- /dev/null +++ b/tests/queries/0_stateless/02293_http_header_summary_contains_exception_code_with_progress.reference @@ -0,0 +1 @@ +Expected exception: 159 diff --git a/tests/queries/0_stateless/02293_http_header_summary_contains_exception_code_with_progress.sh b/tests/queries/0_stateless/02293_http_header_summary_contains_exception_code_with_progress.sh new file mode 100755 index 00000000000..fba136e7c38 --- /dev/null +++ b/tests/queries/0_stateless/02293_http_header_summary_contains_exception_code_with_progress.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +CURL_OUTPUT=$(echo 'SELECT 1 + sleepEachRow(0.00002) FROM numbers(100000)' | \ + ${CLICKHOUSE_CURL_COMMAND} -v "${CLICKHOUSE_URL}&wait_end_of_query=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&max_execution_time=1" --data-binary @- 2>&1) + +EXCEPTION=$(echo "${CURL_OUTPUT}" | grep 'X-ClickHouse-Exception-Code') + +if [[ "$EXCEPTION" =~ .*"159".* ]]; +then + echo "Expected exception: 159" +else + echo "Unexpected exception" + echo "EXCEPTION:" + echo "'${EXCEPTION}'" + echo "DATA:" + echo "$CURL_OUTPUT" +fi + diff --git a/tests/queries/0_stateless/02294_nothing_arguments_in_functions.reference b/tests/queries/0_stateless/02294_nothing_arguments_in_functions.reference new file mode 100644 index 00000000000..665931efb71 --- /dev/null +++ b/tests/queries/0_stateless/02294_nothing_arguments_in_functions.reference @@ -0,0 +1,29 @@ +[] +Array(Nothing) +[] +Array(Nothing) +[] +Array(Nothing) +[] +Array(Nothing) +Array(String) +Array(Nothing) +Array(Nothing) +Array(Array(Nothing)) +Array(Array(Nothing)) +Array(Map(UInt8, Nothing)) +Array(Map(UInt8, Nothing)) +Array(Tuple(Nothing)) +Array(Tuple(UInt8, Nothing)) +Nothing +Nothing +Nothing +Nothing +Array(Nothing) +Array(Nothing) +Map(UInt8, Nothing) +Map(UInt8, Nothing) +Tuple(UInt8, Nothing) +Tuple(UInt8, Nothing) +Nothing +Nothing diff --git a/tests/queries/0_stateless/02294_nothing_arguments_in_functions.sql b/tests/queries/0_stateless/02294_nothing_arguments_in_functions.sql new file mode 100644 index 00000000000..4406a05df0c --- /dev/null +++ b/tests/queries/0_stateless/02294_nothing_arguments_in_functions.sql @@ -0,0 +1,40 @@ +select arrayMap(x -> 2 * x, []); +select toTypeName(arrayMap(x -> 2 * x, [])); +select arrayMap((x, y) -> x + y, [], []); +select toTypeName(arrayMap((x, y) -> x + y, [], [])); +select arrayMap((x, y) -> x + y, [], CAST([], 'Array(Int32)')); +select toTypeName(arrayMap((x, y) -> x + y, [], CAST([], 'Array(Int32)'))); + +select arrayFilter(x -> 2 * x < 0, []); +select toTypeName(arrayFilter(x -> 2 * x < 0, [])); + +select toTypeName(arrayMap(x -> CAST(x, 'String'), [])); +select toTypeName(arrayMap(x -> toInt32(x), [])); +select toColumnTypeName(arrayMap(x -> toInt32(x), [])); + +select toTypeName(arrayMap(x -> [x], [])); +select toColumnTypeName(arrayMap(x -> [x], [])); + +select toTypeName(arrayMap(x ->map(1, x), [])); +select toColumnTypeName(arrayMap(x -> map(1, x), [])); + +select toTypeName(arrayMap(x ->tuple(x), [])); +select toColumnTypeName(arrayMap(x -> tuple(1, x), [])); + +select toTypeName(toInt32(assumeNotNull(materialize(NULL)))); +select toColumnTypeName(toInt32(assumeNotNull(materialize(NULL)))); + +select toTypeName(assumeNotNull(materialize(NULL))); +select toColumnTypeName(assumeNotNull(materialize(NULL))); + +select toTypeName([assumeNotNull(materialize(NULL))]); +select toColumnTypeName([assumeNotNull(materialize(NULL))]); + +select toTypeName(map(1, assumeNotNull(materialize(NULL)))); +select toColumnTypeName(map(1, assumeNotNull(materialize(NULL)))); + +select toTypeName(tuple(1, assumeNotNull(materialize(NULL)))); +select toColumnTypeName(tuple(1, assumeNotNull(materialize(NULL)))); + +select toTypeName(assumeNotNull(materialize(NULL)) * 2); +select toColumnTypeName(assumeNotNull(materialize(NULL)) * 2); diff --git a/tests/queries/0_stateless/02294_nothing_arguments_in_functions_errors.reference b/tests/queries/0_stateless/02294_nothing_arguments_in_functions_errors.reference new file mode 100644 index 00000000000..0eabe367130 --- /dev/null +++ b/tests/queries/0_stateless/02294_nothing_arguments_in_functions_errors.reference @@ -0,0 +1,3 @@ +OK +OK +OK diff --git a/tests/queries/0_stateless/02294_nothing_arguments_in_functions_errors.sh b/tests/queries/0_stateless/02294_nothing_arguments_in_functions_errors.sh new file mode 100755 index 00000000000..931985340c2 --- /dev/null +++ b/tests/queries/0_stateless/02294_nothing_arguments_in_functions_errors.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL -q "SELECT assumeNotNull(NULL)" 2>&1 | grep -q "ILLEGAL_COLUMN" && echo "OK" || echo "FAIL" +$CLICKHOUSE_LOCAL -q "SELECT assumeNotNull(materialize(NULL))" 2>&1 | grep -q "ILLEGAL_TYPE_OF_ARGUMENT" && echo "OK" || echo "FAIL" +$CLICKHOUSE_LOCAL -q "SELECT assumeNotNull(materialize(NULL)) from numbers(10)" 2>&1 | grep -q "ILLEGAL_TYPE_OF_ARGUMENT" && echo "OK" || echo "FAIL" + diff --git a/tests/queries/0_stateless/02294_stringsearch_with_nonconst_needle.reference b/tests/queries/0_stateless/02294_stringsearch_with_nonconst_needle.reference new file mode 100644 index 00000000000..7471bcad00c --- /dev/null +++ b/tests/queries/0_stateless/02294_stringsearch_with_nonconst_needle.reference @@ -0,0 +1,190 @@ +LIKE +1 Hello 0 +2 Hello % 1 +3 Hello %% 1 +4 Hello %%% 1 +5 Hello %_% 1 +6 Hello _ 0 +7 Hello _% 1 +8 Hello %_ 1 +9 Hello H%o 1 +10 hello H%0 0 +11 hello h%o 1 +12 Hello h%o 0 +13 OHello %lhell% 0 +14 OHello %hell% 0 +15 hEllo %HEL% 0 +16 abcdef %aBc%def% 0 +17 ABCDDEF %abc%def% 0 +18 Abc\nDef %abc%def% 0 +19 abc\ntdef %abc%def% 1 +20 abct\ndef %abc%dEf% 0 +21 abc\n\ndeF %abc%def% 0 +22 abc\n\ntdef %abc%deF% 0 +23 Abc\nt\ndef %abc%def% 0 +24 abct\n\ndef %abc%def% 1 +25 ab\ndef %Abc%def% 0 +26 aBc\nef %ABC%DEF% 0 +27 ёЁё Ё%Ё 0 +28 ощщЁё Щ%Ё 0 +29 ощЩЁё %Щ%Ё 0 +30 Щущпандер %щп%е% 1 +31 Щущпандер %щП%е% 0 +32 ощщЁё %щ% 1 +33 ощЩЁё %ё% 1 +34 Hello .* 0 +35 Hello .*ell.* 0 +36 Hello o$ 0 +37 Hello hE.*lO 0 +NOT LIKE +1 Hello 1 +2 Hello % 0 +3 Hello %% 0 +4 Hello %%% 0 +5 Hello %_% 0 +6 Hello _ 1 +7 Hello _% 0 +8 Hello %_ 0 +9 Hello H%o 0 +10 hello H%0 1 +11 hello h%o 0 +12 Hello h%o 1 +13 OHello %lhell% 1 +14 OHello %hell% 1 +15 hEllo %HEL% 1 +16 abcdef %aBc%def% 1 +17 ABCDDEF %abc%def% 1 +18 Abc\nDef %abc%def% 1 +19 abc\ntdef %abc%def% 0 +20 abct\ndef %abc%dEf% 1 +21 abc\n\ndeF %abc%def% 1 +22 abc\n\ntdef %abc%deF% 1 +23 Abc\nt\ndef %abc%def% 1 +24 abct\n\ndef %abc%def% 0 +25 ab\ndef %Abc%def% 1 +26 aBc\nef %ABC%DEF% 1 +27 ёЁё Ё%Ё 1 +28 ощщЁё Щ%Ё 1 +29 ощЩЁё %Щ%Ё 1 +30 Щущпандер %щп%е% 0 +31 Щущпандер %щП%е% 1 +32 ощщЁё %щ% 0 +33 ощЩЁё %ё% 0 +34 Hello .* 1 +35 Hello .*ell.* 1 +36 Hello o$ 1 +37 Hello hE.*lO 1 +ILIKE +1 Hello 0 +2 Hello % 1 +3 Hello %% 1 +4 Hello %%% 1 +5 Hello %_% 1 +6 Hello _ 0 +7 Hello _% 1 +8 Hello %_ 1 +9 Hello H%o 1 +10 hello H%0 0 +11 hello h%o 1 +12 Hello h%o 1 +13 OHello %lhell% 0 +14 OHello %hell% 1 +15 hEllo %HEL% 1 +16 abcdef %aBc%def% 1 +17 ABCDDEF %abc%def% 1 +18 Abc\nDef %abc%def% 1 +19 abc\ntdef %abc%def% 1 +20 abct\ndef %abc%dEf% 1 +21 abc\n\ndeF %abc%def% 1 +22 abc\n\ntdef %abc%deF% 1 +23 Abc\nt\ndef %abc%def% 1 +24 abct\n\ndef %abc%def% 1 +25 ab\ndef %Abc%def% 0 +26 aBc\nef %ABC%DEF% 0 +27 ёЁё Ё%Ё 1 +28 ощщЁё Щ%Ё 0 +29 ощЩЁё %Щ%Ё 1 +30 Щущпандер %щп%е% 1 +31 Щущпандер %щП%е% 1 +32 ощщЁё %щ% 1 +33 ощЩЁё %ё% 1 +34 Hello .* 0 +35 Hello .*ell.* 0 +36 Hello o$ 0 +37 Hello hE.*lO 0 +NOT ILIKE +1 Hello 1 +2 Hello % 0 +3 Hello %% 0 +4 Hello %%% 0 +5 Hello %_% 0 +6 Hello _ 1 +7 Hello _% 0 +8 Hello %_ 0 +9 Hello H%o 0 +10 hello H%0 1 +11 hello h%o 0 +12 Hello h%o 0 +13 OHello %lhell% 1 +14 OHello %hell% 0 +15 hEllo %HEL% 0 +16 abcdef %aBc%def% 0 +17 ABCDDEF %abc%def% 0 +18 Abc\nDef %abc%def% 0 +19 abc\ntdef %abc%def% 0 +20 abct\ndef %abc%dEf% 0 +21 abc\n\ndeF %abc%def% 0 +22 abc\n\ntdef %abc%deF% 0 +23 Abc\nt\ndef %abc%def% 0 +24 abct\n\ndef %abc%def% 0 +25 ab\ndef %Abc%def% 1 +26 aBc\nef %ABC%DEF% 1 +27 ёЁё Ё%Ё 0 +28 ощщЁё Щ%Ё 1 +29 ощЩЁё %Щ%Ё 0 +30 Щущпандер %щп%е% 0 +31 Щущпандер %щП%е% 0 +32 ощщЁё %щ% 0 +33 ощЩЁё %ё% 0 +34 Hello .* 1 +35 Hello .*ell.* 1 +36 Hello o$ 1 +37 Hello hE.*lO 1 +MATCH +1 Hello 1 +2 Hello % 0 +3 Hello %% 0 +4 Hello %%% 0 +5 Hello %_% 0 +6 Hello _ 0 +7 Hello _% 0 +8 Hello %_ 0 +9 Hello H%o 0 +10 hello H%0 0 +11 hello h%o 0 +12 Hello h%o 0 +13 OHello %lhell% 0 +14 OHello %hell% 0 +15 hEllo %HEL% 0 +16 abcdef %aBc%def% 0 +17 ABCDDEF %abc%def% 0 +18 Abc\nDef %abc%def% 0 +19 abc\ntdef %abc%def% 0 +20 abct\ndef %abc%dEf% 0 +21 abc\n\ndeF %abc%def% 0 +22 abc\n\ntdef %abc%deF% 0 +23 Abc\nt\ndef %abc%def% 0 +24 abct\n\ndef %abc%def% 0 +25 ab\ndef %Abc%def% 0 +26 aBc\nef %ABC%DEF% 0 +27 ёЁё Ё%Ё 0 +28 ощщЁё Щ%Ё 0 +29 ощЩЁё %Щ%Ё 0 +30 Щущпандер %щп%е% 0 +31 Щущпандер %щП%е% 0 +32 ощщЁё %щ% 0 +33 ощЩЁё %ё% 0 +34 Hello .* 1 +35 Hello .*ell.* 1 +36 Hello o$ 1 +37 Hello hE.*lO 0 diff --git a/tests/queries/0_stateless/02294_stringsearch_with_nonconst_needle.sql b/tests/queries/0_stateless/02294_stringsearch_with_nonconst_needle.sql new file mode 100644 index 00000000000..3057e342733 --- /dev/null +++ b/tests/queries/0_stateless/02294_stringsearch_with_nonconst_needle.sql @@ -0,0 +1,36 @@ +drop table if exists non_const_needle; + +create table non_const_needle + (id UInt32, haystack String, needle String) + engine = MergeTree() + order by id; + +-- 1 - 33: LIKE-syntax, 34-37: re2-syntax +insert into non_const_needle values (1, 'Hello', '') (2, 'Hello', '%') (3, 'Hello', '%%') (4, 'Hello', '%%%') (5, 'Hello', '%_%') (6, 'Hello', '_') (7, 'Hello', '_%') (8, 'Hello', '%_') (9, 'Hello', 'H%o') (10, 'hello', 'H%0') (11, 'hello', 'h%o') (12, 'Hello', 'h%o') (13, 'OHello', '%lhell%') (14, 'OHello', '%hell%') (15, 'hEllo', '%HEL%') (16, 'abcdef', '%aBc%def%') (17, 'ABCDDEF', '%abc%def%') (18, 'Abc\nDef', '%abc%def%') (19, 'abc\ntdef', '%abc%def%') (20, 'abct\ndef', '%abc%dEf%') (21, 'abc\n\ndeF', '%abc%def%') (22, 'abc\n\ntdef', '%abc%deF%') (23, 'Abc\nt\ndef', '%abc%def%') (24, 'abct\n\ndef', '%abc%def%') (25, 'ab\ndef', '%Abc%def%') (26, 'aBc\nef', '%ABC%DEF%') (27, 'ёЁё', 'Ё%Ё') (28, 'ощщЁё', 'Щ%Ё') (29, 'ощЩЁё', '%Щ%Ё') (30, 'Щущпандер', '%щп%е%') (31, 'Щущпандер', '%щП%е%') (32, 'ощщЁё', '%щ%') (33, 'ощЩЁё', '%ё%') (34, 'Hello', '.*') (35, 'Hello', '.*ell.*') (36, 'Hello', 'o$') (37, 'Hello', 'hE.*lO'); + +select 'LIKE'; +select id, haystack, needle, like(haystack, needle) + from non_const_needle + order by id; + +select 'NOT LIKE'; +select id, haystack, needle, not like(haystack, needle) + from non_const_needle + order by id; + +select 'ILIKE'; +select id, haystack, needle, ilike(haystack, needle) + from non_const_needle + order by id; + +select 'NOT ILIKE'; +select id, haystack, needle, not ilike(haystack, needle) + from non_const_needle + order by id; + +select 'MATCH'; +select id, haystack, needle, match(haystack, needle) + from non_const_needle + order by id; + +drop table if exists non_const_needle; diff --git a/tests/queries/0_stateless/02296_nullable_arguments_in_array_filter.reference b/tests/queries/0_stateless/02296_nullable_arguments_in_array_filter.reference new file mode 100644 index 00000000000..8c11de86262 --- /dev/null +++ b/tests/queries/0_stateless/02296_nullable_arguments_in_array_filter.reference @@ -0,0 +1,4 @@ +[] +[] +[2,4] +[1,3] diff --git a/tests/queries/0_stateless/02296_nullable_arguments_in_array_filter.sql b/tests/queries/0_stateless/02296_nullable_arguments_in_array_filter.sql new file mode 100644 index 00000000000..3c1f2b41909 --- /dev/null +++ b/tests/queries/0_stateless/02296_nullable_arguments_in_array_filter.sql @@ -0,0 +1,4 @@ +select arrayFilter(x -> 2 * x > 0, []); +select arrayFilter(x -> 2 * x > 0, [NULL]); +select arrayFilter(x -> x % 2 ? NULL : 1, [1, 2, 3, 4]); +select arrayFilter(x -> x % 2, [1, NULL, 3, NULL]); diff --git a/tests/queries/0_stateless/02302_column_decl_null_before_defaul_value.reference b/tests/queries/0_stateless/02302_column_decl_null_before_defaul_value.reference new file mode 100644 index 00000000000..2079872ee73 --- /dev/null +++ b/tests/queries/0_stateless/02302_column_decl_null_before_defaul_value.reference @@ -0,0 +1,22 @@ +create table, column +type +NULL +id Nullable(Int32) +create table, column +type +NOT NULL +id Int32 +create table, column +type +NULL +DEFAULT +id Nullable(Int32) DEFAULT 1 +create table, column +type +NOT NULL +DEFAULT +id Int32 DEFAULT 1 +create table, column +type +DEFAULT +NULL +id Nullable(Int32) DEFAULT 1 +create table, column +type +DEFAULT +NOT NULL +id Int32 DEFAULT 1 +create table, column -type +NULL +DEFAULT +id Nullable(UInt8) DEFAULT 1 +create table, column -type +NOT NULL +DEFAULT +id UInt8 DEFAULT 1 +create table, column -type +DEFAULT +NULL +id Nullable(UInt8) DEFAULT 1 +create table, column -type +DEFAULT +NOT NULL +id UInt8 DEFAULT 1 +alter column, NULL modifier is not allowed +modify column, NULL modifier is not allowed diff --git a/tests/queries/0_stateless/02302_column_decl_null_before_defaul_value.sql b/tests/queries/0_stateless/02302_column_decl_null_before_defaul_value.sql new file mode 100644 index 00000000000..3825df1e557 --- /dev/null +++ b/tests/queries/0_stateless/02302_column_decl_null_before_defaul_value.sql @@ -0,0 +1,61 @@ +select 'create table, column +type +NULL'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id INT NULL) ENGINE=MergeTree() ORDER BY tuple(); +DESCRIBE TABLE null_before; + +select 'create table, column +type +NOT NULL'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id INT NOT NULL) ENGINE=MergeTree() ORDER BY tuple(); +DESCRIBE TABLE null_before; + +select 'create table, column +type +NULL +DEFAULT'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id INT NULL DEFAULT 1) ENGINE=MergeTree() ORDER BY tuple(); +DESCRIBE TABLE null_before; + +select 'create table, column +type +NOT NULL +DEFAULT'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id INT NOT NULL DEFAULT 1) ENGINE=MergeTree() ORDER BY tuple(); +DESCRIBE TABLE null_before; + +select 'create table, column +type +DEFAULT +NULL'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id INT DEFAULT 1 NULL) ENGINE=MergeTree() ORDER BY tuple(); +DESCRIBE TABLE null_before; + +select 'create table, column +type +DEFAULT +NOT NULL'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id INT DEFAULT 1 NOT NULL) ENGINE=MergeTree() ORDER BY tuple(); +DESCRIBE TABLE null_before; + +select 'create table, column -type +NULL +DEFAULT'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id NULL DEFAULT 1) ENGINE=MergeTree() ORDER BY tuple(); +DESCRIBE TABLE null_before; + +select 'create table, column -type +NOT NULL +DEFAULT'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id NOT NULL DEFAULT 1) ENGINE=MergeTree() ORDER BY tuple(); +DESCRIBE TABLE null_before; + +select 'create table, column -type +DEFAULT +NULL'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id DEFAULT 1 NULL) ENGINE=MergeTree() ORDER BY tuple(); +DESCRIBE TABLE null_before; + +select 'create table, column -type +DEFAULT +NOT NULL'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id DEFAULT 1 NOT NULL) ENGINE=MergeTree() ORDER BY tuple(); +DESCRIBE TABLE null_before; + +select 'alter column, NULL modifier is not allowed'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id INT NOT NULL) ENGINE=MergeTree() ORDER BY tuple(); +ALTER TABLE null_before ALTER COLUMN id TYPE INT NULL; -- { clientError SYNTAX_ERROR } + +select 'modify column, NULL modifier is not allowed'; +DROP TABLE IF EXISTS null_before SYNC; +CREATE TABLE null_before (id INT NOT NULL) ENGINE=MergeTree() ORDER BY tuple(); +ALTER TABLE null_before MODIFY COLUMN id NULL DEFAULT 1; -- { serverError UNKNOWN_TYPE } + +DROP TABLE IF EXISTS null_before SYNC; diff --git a/tests/queries/0_stateless/02302_join_auto_lc_nullable_bug.reference b/tests/queries/0_stateless/02302_join_auto_lc_nullable_bug.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02302_join_auto_lc_nullable_bug.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02302_join_auto_lc_nullable_bug.sql b/tests/queries/0_stateless/02302_join_auto_lc_nullable_bug.sql new file mode 100644 index 00000000000..469476c82bf --- /dev/null +++ b/tests/queries/0_stateless/02302_join_auto_lc_nullable_bug.sql @@ -0,0 +1,7 @@ +-- Tags: no-backward-compatibility-check + +SET max_bytes_in_join = '100', join_algorithm = 'auto'; + +SELECT 3 == count() FROM (SELECT toLowCardinality(toNullable(number)) AS l FROM system.numbers LIMIT 3) AS s1 +ANY LEFT JOIN (SELECT toLowCardinality(toNullable(number)) AS r FROM system.numbers LIMIT 4) AS s2 ON l = r +; diff --git a/tests/queries/0_stateless/02302_s3_file_pruning.reference b/tests/queries/0_stateless/02302_s3_file_pruning.reference new file mode 100644 index 00000000000..f8d2bdd0612 --- /dev/null +++ b/tests/queries/0_stateless/02302_s3_file_pruning.reference @@ -0,0 +1,27 @@ +-- { echo } +drop table if exists test_02302; +create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302_{_partition_id}', format=Parquet) partition by a; +insert into test_02302 select number from numbers(10) settings s3_truncate_on_insert=1; +select * from test_02302; -- { serverError 48 } +drop table test_02302; +set max_rows_to_read = 1; +-- Test s3 table function with glob +select * from s3(s3_conn, filename='test_02302_*', format=Parquet) where _file like '%5'; +5 +-- Test s3 table with explicit keys (no glob) +-- TODO support truncate table function +drop table if exists test_02302; +create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302.2', format=Parquet); +truncate table test_02302; +drop table if exists test_02302; +create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302.1', format=Parquet); +truncate table test_02302; +drop table if exists test_02302; +create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302', format=Parquet); +truncate table test_02302; +insert into test_02302 select 0 settings s3_create_new_file_on_insert = true; +insert into test_02302 select 1 settings s3_create_new_file_on_insert = true; +insert into test_02302 select 2 settings s3_create_new_file_on_insert = true; +select * from test_02302 where _file like '%1'; +1 +drop table test_02302; diff --git a/tests/queries/0_stateless/02302_s3_file_pruning.sql b/tests/queries/0_stateless/02302_s3_file_pruning.sql new file mode 100644 index 00000000000..624a87506d1 --- /dev/null +++ b/tests/queries/0_stateless/02302_s3_file_pruning.sql @@ -0,0 +1,35 @@ +-- Tags: no-parallel, no-fasttest +-- Tag no-fasttest: Depends on AWS + +-- { echo } +drop table if exists test_02302; +create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302_{_partition_id}', format=Parquet) partition by a; +insert into test_02302 select number from numbers(10) settings s3_truncate_on_insert=1; +select * from test_02302; -- { serverError 48 } +drop table test_02302; + +set max_rows_to_read = 1; + +-- Test s3 table function with glob +select * from s3(s3_conn, filename='test_02302_*', format=Parquet) where _file like '%5'; + +-- Test s3 table with explicit keys (no glob) +-- TODO support truncate table function +drop table if exists test_02302; +create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302.2', format=Parquet); +truncate table test_02302; + +drop table if exists test_02302; +create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302.1', format=Parquet); +truncate table test_02302; + +drop table if exists test_02302; +create table test_02302 (a UInt64) engine = S3(s3_conn, filename='test_02302', format=Parquet); +truncate table test_02302; + +insert into test_02302 select 0 settings s3_create_new_file_on_insert = true; +insert into test_02302 select 1 settings s3_create_new_file_on_insert = true; +insert into test_02302 select 2 settings s3_create_new_file_on_insert = true; + +select * from test_02302 where _file like '%1'; +drop table test_02302; diff --git a/tests/queries/0_stateless/02311_normalize_utf8_constant.reference b/tests/queries/0_stateless/02311_normalize_utf8_constant.reference new file mode 100644 index 00000000000..efd3caf8a45 --- /dev/null +++ b/tests/queries/0_stateless/02311_normalize_utf8_constant.reference @@ -0,0 +1 @@ +â â â â â C3A2 C3A2 61CC82 C3A2 61CC82 diff --git a/tests/queries/0_stateless/02311_normalize_utf8_constant.sql b/tests/queries/0_stateless/02311_normalize_utf8_constant.sql new file mode 100644 index 00000000000..2747aa073ec --- /dev/null +++ b/tests/queries/0_stateless/02311_normalize_utf8_constant.sql @@ -0,0 +1,13 @@ +-- Tags: no-fasttest + +SELECT + 'â' AS s, + normalizeUTF8NFC(s) s1, + normalizeUTF8NFD(s) s2, + normalizeUTF8NFKC(s) s3, + normalizeUTF8NFKD(s) s4, + hex(s), + hex(s1), + hex(s2), + hex(s3), + hex(s4); diff --git a/tests/queries/0_stateless/02312_parquet_orc_arrow_names_tuples.reference b/tests/queries/0_stateless/02312_parquet_orc_arrow_names_tuples.reference new file mode 100644 index 00000000000..4697d53a23b --- /dev/null +++ b/tests/queries/0_stateless/02312_parquet_orc_arrow_names_tuples.reference @@ -0,0 +1,12 @@ +(1,2) +(2,3) +(3,4) +(1,2) +(2,3) +(3,4) +(1,2) +(2,3) +(3,4) +[[(1),(2),(3)]] +[[(1),(2),(3)]] +[[(1),(2),(3)]] diff --git a/tests/queries/0_stateless/02312_parquet_orc_arrow_names_tuples.sql b/tests/queries/0_stateless/02312_parquet_orc_arrow_names_tuples.sql new file mode 100644 index 00000000000..4c2158e4a0c --- /dev/null +++ b/tests/queries/0_stateless/02312_parquet_orc_arrow_names_tuples.sql @@ -0,0 +1,29 @@ +-- Tags: no-fasttest + +drop table if exists test_02312; +create table test_02312 (x Tuple(a UInt32, b UInt32)) engine=File(Parquet); +insert into test_02312 values ((1,2)), ((2,3)), ((3,4)); +select * from test_02312; +drop table test_02312; +create table test_02312 (x Tuple(a UInt32, b UInt32)) engine=File(Arrow); +insert into test_02312 values ((1,2)), ((2,3)), ((3,4)); +select * from test_02312; +drop table test_02312; +create table test_02312 (x Tuple(a UInt32, b UInt32)) engine=File(ORC); +insert into test_02312 values ((1,2)), ((2,3)), ((3,4)); +select * from test_02312; +drop table test_02312; + +create table test_02312 (a Nested(b Nested(c UInt32))) engine=File(Parquet); +insert into test_02312 values ([[(1), (2), (3)]]); +select * from test_02312; +drop table test_02312; +create table test_02312 (a Nested(b Nested(c UInt32))) engine=File(Arrow); +insert into test_02312 values ([[(1), (2), (3)]]); +select * from test_02312; +drop table test_02312; +create table test_02312 (a Nested(b Nested(c UInt32))) engine=File(ORC); +insert into test_02312 values ([[(1), (2), (3)]]); +select * from test_02312; +drop table test_02312; + diff --git a/utils/changelog/.gitignore b/utils/changelog/.gitignore index 9ab24b6c8b8..f0a3171ed34 100644 --- a/utils/changelog/.gitignore +++ b/utils/changelog/.gitignore @@ -1,3 +1,4 @@ *.md *.txt *.json +gh_cache diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index c15a1600506..c20c6cfd072 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -3,8 +3,10 @@ import argparse import logging +import os.path as p +import os import re -from datetime import date, timedelta +from datetime import date, datetime, timedelta from queue import Empty, Queue from subprocess import CalledProcessError, DEVNULL from threading import Thread @@ -13,6 +15,7 @@ from typing import Dict, List, Optional, TextIO from fuzzywuzzy.fuzz import ratio # type: ignore from github import Github from github.NamedUser import NamedUser +from github.Issue import Issue from github.PullRequest import PullRequest from github.Repository import Repository from git_helper import is_shallow, git_runner as runner @@ -34,6 +37,8 @@ categories_preferred_order = ( FROM_REF = "" TO_REF = "" SHA_IN_CHANGELOG = [] # type: List[str] +GitHub = Github() +CACHE_PATH = p.join(p.dirname(p.realpath(__file__)), "gh_cache") class Description: @@ -87,10 +92,10 @@ class Worker(Thread): def run(self): while not self.queue.empty(): try: - number = self.queue.get() + issue = self.queue.get() # type: Issue except Empty: break # possible race condition, just continue - api_pr = self.repo.get_pull(number) + api_pr = get_pull_cached(self.repo, issue.number, issue.updated_at) in_changelog = False merge_commit = api_pr.merge_commit_sha try: @@ -109,13 +114,31 @@ class Worker(Thread): self.queue.task_done() +def get_pull_cached( + repo: Repository, number: int, updated_at: Optional[datetime] = None +) -> PullRequest: + pr_cache_file = p.join(CACHE_PATH, f"{number}.pickle") + if updated_at is None: + updated_at = datetime.now() - timedelta(hours=-1) + + if p.isfile(pr_cache_file): + cache_updated = datetime.fromtimestamp(p.getmtime(pr_cache_file)) + if cache_updated > updated_at: + with open(pr_cache_file, "rb") as prfd: + return GitHub.load(prfd) # type: ignore + pr = repo.get_pull(number) + with open(pr_cache_file, "wb") as prfd: + GitHub.dump(pr, prfd) # type: ignore + return pr + + def get_descriptions( - repo: Repository, numbers: List[int], jobs: int + repo: Repository, issues: List[Issue], jobs: int ) -> Dict[str, List[Description]]: workers = [] # type: List[Worker] - queue = Queue() # type: Queue # (!?!?!?!??!) - for number in numbers: - queue.put(number) + queue = Queue() # type: Queue[Issue] + for issue in issues: + queue.put(issue) for _ in range(jobs): worker = Worker(queue, repo) worker.start() @@ -173,6 +196,11 @@ def parse_args() -> argparse.Namespace: "--gh-password", help="a password that should be used when user is given", ) + parser.add_argument( + "--with-testing-tags", + action="store_true", + help="by default '*-testing' tags are ignored, this argument enables them too", + ) parser.add_argument( "--from", dest="from_ref", @@ -195,7 +223,10 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri if item.head.ref.startswith("backport/"): branch_parts = item.head.ref.split("/") if len(branch_parts) == 3: - item = repo.get_pull(int(branch_parts[-1])) + try: + item = get_pull_cached(repo, int(branch_parts[-1])) + except Exception as e: + logging.warning("unable to get backpoted PR, exception: %s", e) else: logging.warning( "The branch %s doesn't match backport template, using PR %s as is", @@ -296,7 +327,7 @@ def write_changelog(fd: TextIO, descriptions: Dict[str, List[Description]]): fd.write("\n") -def check_refs(from_ref: Optional[str], to_ref: str): +def check_refs(from_ref: Optional[str], to_ref: str, with_testing_tags: bool): global FROM_REF, TO_REF TO_REF = to_ref @@ -306,10 +337,13 @@ def check_refs(from_ref: Optional[str], to_ref: str): # Check from_ref if from_ref is None: # Get all tags pointing to TO_REF - tags = runner.run(f"git tag --points-at '{TO_REF}^{{}}'") + tags = runner.run(f"git tag --points-at '{TO_REF}^{{}}'").split("\n") logging.info("All tags pointing to %s:\n%s", TO_REF, tags) - exclude = " ".join([f"--exclude='{tag}'" for tag in tags.split("\n")]) - FROM_REF = runner.run(f"git describe --abbrev=0 --tags {exclude} '{TO_REF}'") + if not with_testing_tags: + tags.append("*-testing") + exclude = " ".join([f"--exclude='{tag}'" for tag in tags]) + cmd = f"git describe --abbrev=0 --tags {exclude} '{TO_REF}'" + FROM_REF = runner.run(cmd) else: runner.run(f"git rev-parse {FROM_REF}") FROM_REF = from_ref @@ -329,6 +363,10 @@ def main(): format="%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d]:\n%(message)s", level=log_levels[min(args.verbose, 3)], ) + # Create a cache directory + if not p.isdir(CACHE_PATH): + os.mkdir(CACHE_PATH, 0o700) + # Get the full repo if is_shallow(): logging.info("Unshallow repository") @@ -336,7 +374,7 @@ def main(): logging.info("Fetching all tags") runner.run("git fetch --tags", stderr=DEVNULL) - check_refs(args.from_ref, args.to_ref) + check_refs(args.from_ref, args.to_ref, args.with_testing_tags) set_sha_in_changelog() logging.info("Using %s..%s as changelog interval", FROM_REF, TO_REF) @@ -344,21 +382,23 @@ def main(): # Get starting and ending dates for gathering PRs # Add one day after and before to mitigate TZ possible issues # `tag^{}` format gives commit ref when we have annotated tags - from_date = runner.run(f"git log -1 --format=format:%as '{FROM_REF}^{{}}'") + # format %cs gives a committer date, works better for cherry-picked commits + from_date = runner.run(f"git log -1 --format=format:%cs '{FROM_REF}^{{}}'") from_date = (date.fromisoformat(from_date) - timedelta(1)).isoformat() - to_date = runner.run(f"git log -1 --format=format:%as '{TO_REF}^{{}}'") + to_date = runner.run(f"git log -1 --format=format:%cs '{TO_REF}^{{}}'") to_date = (date.fromisoformat(to_date) + timedelta(1)).isoformat() # Get all PRs for the given time frame - gh = Github( + global GitHub + GitHub = Github( args.gh_user_or_token, args.gh_password, per_page=100, pool_size=args.jobs ) query = f"type:pr repo:{args.repo} is:merged merged:{from_date}..{to_date}" - repo = gh.get_repo(args.repo) - api_prs = gh.search_issues(query=query, sort="created") + repo = GitHub.get_repo(args.repo) + api_prs = GitHub.search_issues(query=query, sort="created") logging.info("Found %s PRs for the query: '%s'", api_prs.totalCount, query) - pr_numbers = [pr.number for pr in api_prs] + pr_numbers = list(api_prs) descriptions = get_descriptions(repo, pr_numbers, args.jobs) diff --git a/website/README.md b/website/README.md index 57cd87cbfe2..f96f1e0075d 100644 --- a/website/README.md +++ b/website/README.md @@ -22,19 +22,7 @@ pip3 install -r requirements.txt virtualenv build -./build.py --skip-multi-page --skip-blog --skip-docs --livereload 8080 +./build.py --livereload 8080 # Open the web browser and go to http://localhost:8080/ ``` - -# How to quickly test the blog - -``` -./build.py --skip-multi-page --skip-docs --livereload 8080 -``` - -# How to quickly test the broken links in docs - -``` -./build.py --skip-multi-page --skip-blog --lang en --livereload 8080 -``` diff --git a/website/benchmark/hardware/results/aws_c6g_16xlarge.json b/website/benchmark/hardware/results/aws_c6g_16xlarge.json new file mode 100644 index 00000000000..364b40f657a --- /dev/null +++ b/website/benchmark/hardware/results/aws_c6g_16xlarge.json @@ -0,0 +1,54 @@ +[ + { + "system": "AWS c6g.16xlarge (Graviton 2)", + "system_full": "AWS c6g.16xlarge (Graviton 2) 64 vCPU, 128 GiB RAM, EBS", + "time": "2022-05-24 00:00:00", + "kind": "cloud", + "result": + [ +[0.002, 0.002, 0.002], +[0.051, 0.024, 0.026], +[0.037, 0.021, 0.028], +[0.102, 0.065, 0.061], +[0.243, 0.080, 0.080], +[0.976, 0.138, 0.138], +[0.003, 0.002, 0.002], +[0.044, 0.040, 0.039], +[0.204, 0.145, 0.146], +[0.799, 0.165, 0.165], +[0.306, 0.095, 0.095], +[0.523, 0.101, 0.096], +[0.973, 0.226, 0.224], +[1.520, 0.282, 0.277], +[0.645, 0.239, 0.236], +[0.260, 0.312, 0.280], +[1.535, 0.660, 0.629], +[1.426, 0.470, 0.427], +[3.456, 1.372, 1.138], +[0.147, 0.119, 0.079], +[9.101, 0.406, 0.358], +[10.117, 0.330, 0.323], +[19.495, 0.756, 0.748], +[16.173, 1.500, 1.532], +[1.832, 0.105, 0.094], +[0.836, 0.092, 0.090], +[2.363, 0.108, 0.099], +[9.269, 0.367, 0.363], +[7.317, 0.422, 0.414], +[0.918, 1.020, 1.058], +[1.347, 0.210, 0.209], +[4.535, 0.343, 0.335], +[4.288, 2.411, 2.501], +[9.310, 1.240, 1.172], +[9.301, 1.209, 1.205], +[0.446, 0.428, 0.421], +[0.245, 0.207, 0.202], +[0.107, 0.091, 0.098], +[0.112, 0.095, 0.101], +[0.546, 0.485, 0.444], +[0.061, 0.049, 0.037], +[0.041, 0.035, 0.033], +[0.006, 0.005, 0.005] + ] + } +] diff --git a/website/benchmark/hardware/results/aws_c7g_16xlarge.json b/website/benchmark/hardware/results/aws_c7g_16xlarge.json new file mode 100644 index 00000000000..91230ecceee --- /dev/null +++ b/website/benchmark/hardware/results/aws_c7g_16xlarge.json @@ -0,0 +1,54 @@ +[ + { + "system": "AWS c7g.16xlarge (Graviton 3)", + "system_full": "AWS c7g.16xlarge (Graviton 3) 64 vCPU, 128 GiB RAM, EBS", + "time": "2022-05-24 00:00:00", + "kind": "cloud", + "result": + [ +[0.002, 0.002, 0.002], +[0.031, 0.022, 0.023], +[0.066, 0.025, 0.025], +[0.240, 0.061, 0.059], +[0.328, 0.073, 0.076], +[0.955, 0.101, 0.098], +[0.002, 0.002, 0.002], +[0.035, 0.030, 0.030], +[0.499, 0.113, 0.115], +[0.704, 0.127, 0.127], +[0.452, 0.070, 0.070], +[0.613, 0.074, 0.072], +[1.060, 0.147, 0.144], +[1.749, 0.190, 0.187], +[0.933, 0.176, 0.175], +[0.408, 0.206, 0.188], +[1.714, 0.476, 0.464], +[1.391, 0.349, 0.307], +[3.271, 0.876, 0.719], +[0.375, 0.079, 0.071], +[9.094, 0.270, 0.293], +[10.251, 0.236, 0.222], +[19.763, 0.783, 0.839], +[16.380, 1.164, 1.192], +[1.861, 0.112, 0.114], +[0.863, 0.062, 0.060], +[2.499, 0.103, 0.113], +[9.448, 0.257, 0.245], +[7.546, 0.288, 0.285], +[0.822, 0.837, 0.837], +[1.352, 0.151, 0.142], +[4.743, 0.224, 0.214], +[3.807, 1.236, 1.366], +[10.096, 0.805, 0.780], +[9.191, 0.830, 0.792], +[0.320, 0.304, 0.294], +[0.209, 0.143, 0.175], +[0.099, 0.066, 0.068], +[0.141, 0.073, 0.064], +[0.499, 0.386, 0.372], +[0.061, 0.030, 0.032], +[0.035, 0.030, 0.028], +[0.016, 0.016, 0.004] + ] + } +]