Merge branch 'master' into add_delay_for_replicated

.gitignore

@@ -69,6 +69,7 @@ cmake-build-*
 *.pyc
 __pycache__
 *.pytest_cache
+.mypy_cache
 
 test.cpp
 CPackConfig.cmake
@@ -167,3 +168,4 @@ tests/integration/**/_gen
 /rust/**/target
 # It is autogenerated from *.in
 /rust/**/.cargo/config.toml
+/rust/**/vendor

docker/packager/binary/Dockerfile

@@ -58,6 +58,33 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
     rustup target add aarch64-apple-darwin && \
     rustup target add powerpc64le-unknown-linux-gnu
 
+# Create vendor cache for cargo.
+#
+# Note, that the config.toml for the root is used, you will not be able to
+# install any other crates, except those which had been vendored (since if
+# there is "replace-with" for some source, then cargo will not look to other
+# remotes except this).
+#
+# Notes for the command itself:
+# - --chown is required to preserve the rights
+# - unstable-options for -C
+# - chmod is required to fix the permissions, since builds are running from a different user
+# - copy of the Cargo.lock is required for proper dependencies versions
+# - cargo vendor --sync is requried to overcome [1] bug.
+#
+#   [1]: https://github.com/rust-lang/wg-cargo-std-aware/issues/23
+COPY --chown=root:root /rust /rust/packages
+RUN cargo -Z unstable-options -C /rust/packages vendor > $CARGO_HOME/config.toml && \
+    cp "$(rustc --print=sysroot)"/lib/rustlib/src/rust/Cargo.lock "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/ && \
+    cargo -Z unstable-options -C /rust/packages vendor --sync "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/Cargo.toml && \
+    rm "$(rustc --print=sysroot)"/lib/rustlib/src/rust/library/test/Cargo.lock && \
+    sed -i "s#\"vendor\"#\"/rust/vendor\"#" $CARGO_HOME/config.toml && \
+    cat $CARGO_HOME/config.toml && \
+    mv /rust/packages/vendor /rust/vendor && \
+    chmod -R o=r+X /rust/vendor && \
+    ls -R -l /rust/packages && \
+    rm -r /rust/packages
+
 # NOTE: Seems like gcc-11 is too new for ubuntu20 repository
 # A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work):
 RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \

docker/packager/binary/rust

@@ -0,0 +1 @@
+../../../rust

docs/en/sql-reference/functions/string-search-functions.md

@@ -631,3 +631,53 @@ Result:
 │ 100                                          │ 200                                          │ 100-200                                      │ 100                                       │
 └──────────────────────────────────────────────┴──────────────────────────────────────────────┴──────────────────────────────────────────────┴───────────────────────────────────────────┘
 ```
+
+## hasSubsequence
+
+Returns 1 if needle is a subsequence of haystack, or 0 otherwise.
+A subsequence of a string is a sequence that can be derived from the given string by deleting zero or more elements without changing the order of the remaining elements.
+
+
+**Syntax**
+
+``` sql
+hasSubsequence(haystack, needle)
+```
+
+**Arguments**
+
+- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Subsequence to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
+
+**Returned values**
+
+- 1, if needle is a subsequence of haystack.
+- 0, otherwise.
+
+Type: `UInt8`.
+
+**Examples**
+
+``` sql
+SELECT hasSubsequence('garbage', 'arg') ;
+```
+
+Result:
+
+``` text
+┌─hasSubsequence('garbage', 'arg')─┐
+│                                1 │
+└──────────────────────────────────┘
+```
+
+## hasSubsequenceCaseInsensitive
+
+Like [hasSubsequence](#hasSubsequence) but searches case-insensitively.
+
+## hasSubsequenceUTF8
+
+Like [hasSubsequence](#hasSubsequence) but assumes `haystack` and `needle` are UTF-8 encoded strings.
+
+## hasSubsequenceCaseInsensitiveUTF8
+
+Like [hasSubsequenceUTF8](#hasSubsequenceUTF8) but searches case-insensitively.

docs/en/sql-reference/functions/type-conversion-functions.md

@@ -945,44 +945,6 @@ Result:
 └────────────┴───────┘
 ```
 
-## toDecimalString
-
-Converts a numeric value to String with the number of fractional digits in the output specified by the user.
-
-**Syntax**
-
-``` sql
-toDecimalString(number, scale)
-```
-
-**Parameters**
-
-- `number` — Value to be represented as String, [Int, UInt](/docs/en/sql-reference/data-types/int-uint.md), [Float](/docs/en/sql-reference/data-types/float.md), [Decimal](/docs/en/sql-reference/data-types/decimal.md),
-- `scale` — Number of fractional digits, [UInt8](/docs/en/sql-reference/data-types/int-uint.md).
-    * Maximum scale for [Decimal](/docs/en/sql-reference/data-types/decimal.md) and [Int, UInt](/docs/en/sql-reference/data-types/int-uint.md) types is 77 (it is the maximum possible number of significant digits for Decimal),
-    * Maximum scale for [Float](/docs/en/sql-reference/data-types/float.md) is 60.
-
-**Returned value**
-
-- Input value represented as [String](/docs/en/sql-reference/data-types/string.md) with given number of fractional digits (scale).
-    The number is rounded up or down according to common arithmetic in case requested scale is smaller than original number's scale.
-
-**Example**
-
-Query:
-
-``` sql
-SELECT toDecimalString(CAST('64.32', 'Float64'), 5);
-```
-
-Result:
-
-```response
-┌toDecimalString(CAST('64.32', 'Float64'), 5)─┐
-│ 64.32000                                    │
-└─────────────────────────────────────────────┘
-```
-
 ## reinterpretAsUInt(8\|16\|32\|64)
 
 ## reinterpretAsInt(8\|16\|32\|64)

docs/ru/sql-reference/functions/string-search-functions.md

@@ -801,3 +801,55 @@ SELECT countSubstringsCaseInsensitiveUTF8('аБв__АбВ__абв', 'Абв');
 │                                                          3 │
 └────────────────────────────────────────────────────────────┘
 ```
+
+## hasSubsequence(haystack, needle) {#hasSubsequence}
+
+Возвращает 1 если needle является подпоследовательностью haystack, иначе 0.
+
+
+**Синтаксис**
+
+``` sql
+hasSubsequence(haystack, needle)
+```
+
+**Аргументы**
+
+-   `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
+-   `needle` — подпоследовательность, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal).
+
+**Возвращаемые значения**
+
+-   1, если 
+-   0, если подстрока не найдена.
+
+Тип: `UInt8`.
+
+**Примеры**
+
+Запрос:
+
+``` sql
+SELECT hasSubsequence('garbage', 'arg') ;
+```
+
+Результат:
+
+``` text
+┌─hasSubsequence('garbage', 'arg')─┐
+│                                1 │
+└──────────────────────────────────┘
+```
+
+
+## hasSubsequenceCaseInsensitive
+
+Такая же, как и [hasSubsequence](#hasSubsequence), но работает без учета регистра.
+
+## hasSubsequenceUTF8
+
+Такая же, как и [hasSubsequence](#hasSubsequence) при допущении что `haystack` и `needle` содержат набор кодовых точек, представляющий текст в кодировке UTF-8.
+
+## hasSubsequenceCaseInsensitiveUTF8
+
+Такая же, как и [hasSubsequenceUTF8](#hasSubsequenceUTF8), но работает без учета регистра.

docs/ru/sql-reference/functions/type-conversion-functions.md

@@ -762,44 +762,6 @@ SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut;
 └────────────┴───────┘
 ```
 
-## toDecimalString
-
-Принимает любой численный тип первым аргументом, возвращает строковое десятичное представление числа с точностью, заданной вторым аргументом.
-
-**Синтаксис**
-
-``` sql
-toDecimalString(number, scale)
-```
-
-**Параметры**
-
--   `number` — Значение любого числового типа: [Int, UInt](/docs/ru/sql-reference/data-types/int-uint.md), [Float](/docs/ru/sql-reference/data-types/float.md), [Decimal](/docs/ru/sql-reference/data-types/decimal.md),
--   `scale` — Требуемое количество десятичных знаков после запятой, [UInt8](/docs/ru/sql-reference/data-types/int-uint.md).
-    * Значение `scale` для типов [Decimal](/docs/ru/sql-reference/data-types/decimal.md) и [Int, UInt](/docs/ru/sql-reference/data-types/int-uint.md) должно не превышать 77 (так как это наибольшее количество значимых символов для этих типов),
-    * Значение `scale` для типа [Float](/docs/ru/sql-reference/data-types/float.md) не должно превышать 60.
-
-**Возвращаемое значение**
-
--   Строка ([String](/docs/en/sql-reference/data-types/string.md)), представляющая собой десятичное представление входного числа с заданной длиной дробной части.
-    При необходимости число округляется по стандартным правилам арифметики.
-
-**Пример использования**
-
-Запрос:
-
-``` sql
-SELECT toDecimalString(CAST('64.32', 'Float64'), 5);
-```
-
-Результат:
-
-```response
-┌─toDecimalString(CAST('64.32', 'Float64'), 5)┐
-│ 64.32000                                    │
-└─────────────────────────────────────────────┘
-```
-
 ## reinterpretAsUInt(8\|16\|32\|64) {#reinterpretasuint8163264}
 
 ## reinterpretAsInt(8\|16\|32\|64) {#reinterpretasint8163264}

programs/server/Server.cpp

@@ -1659,26 +1659,17 @@ try
         global_context->initializeTraceCollector();
 
         /// Set up server-wide memory profiler (for total memory tracker).
-        if (server_settings.total_memory_profiler_step)
+        UInt64 total_memory_profiler_step = config().getUInt64("total_memory_profiler_step", 0);
+        if (total_memory_profiler_step)
         {
-            total_memory_tracker.setProfilerStep(server_settings.total_memory_profiler_step);
+            total_memory_tracker.setProfilerStep(total_memory_profiler_step);
         }
 
-        if (server_settings.total_memory_tracker_sample_probability > 0.0)
+        double total_memory_tracker_sample_probability = config().getDouble("total_memory_tracker_sample_probability", 0);
+        if (total_memory_tracker_sample_probability > 0.0)
         {
-            total_memory_tracker.setSampleProbability(server_settings.total_memory_tracker_sample_probability);
+            total_memory_tracker.setSampleProbability(total_memory_tracker_sample_probability);
         }
-
-        if (server_settings.total_memory_profiler_sample_min_allocation_size)
-        {
-            total_memory_tracker.setSampleMinAllocationSize(server_settings.total_memory_profiler_sample_min_allocation_size);
-        }
-
-        if (server_settings.total_memory_profiler_sample_max_allocation_size)
-        {
-            total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size);
-        }
-
     }
 #endif
 

rust/.dockerignore

@@ -0,0 +1,4 @@
+# Just in case ignore any cargo stuff (and just in case someone will run this
+# docker build locally with build context using folder root):
+target
+vendor

rust/.gitignore

@@ -0,0 +1,4 @@
+# This is for tar --exclude-vcs-ignores (and just in case someone will run
+# docker build locally with build context created via tar):
+target
+vendor

rust/BLAKE3/Cargo.lock

@@ -1,92 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "_ch_rust_blake3"
-version = "0.1.0"
-dependencies = [
- "blake3",
- "libc",
-]
-
-[[package]]
-name = "arrayref"
-version = "0.3.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
-
-[[package]]
-name = "arrayvec"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
-
-[[package]]
-name = "blake3"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "526c210b4520e416420759af363083471656e819a75e831b8d2c9d5a584f2413"
-dependencies = [
- "arrayref",
- "arrayvec",
- "cc",
- "cfg-if",
- "constant_time_eq",
- "digest",
-]
-
-[[package]]
-name = "cc"
-version = "1.0.73"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "constant_time_eq"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
-
-[[package]]
-name = "digest"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
-dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "generic-array"
-version = "0.14.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
-dependencies = [
- "typenum",
- "version_check",
-]
-
-[[package]]
-name = "libc"
-version = "0.2.132"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5"
-
-[[package]]
-name = "typenum"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
-
-[[package]]
-name = "version_check"
-version = "0.9.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"

rust/CMakeLists.txt

@@ -55,6 +55,8 @@ function(clickhouse_import_crate)
         endif()
     endif()
 
+    # Note, here --offline is not used, since on CI vendor archive is used, and
+    # passing --offline here will be inconvenient for local development.
     corrosion_import_crate(NO_STD ${ARGN} PROFILE ${profile})
 endfunction()
 

rust/Cargo.lock

@@ -2,6 +2,22 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "_ch_rust_blake3"
+version = "0.1.0"
+dependencies = [
+ "blake3",
+ "libc",
+]
+
+[[package]]
+name = "_ch_rust_prql"
+version = "0.1.0"
+dependencies = [
+ "prql-compiler",
+ "serde_json",
+]
+
 [[package]]
 name = "_ch_rust_skim_rust"
 version = "0.1.0"
@@ -12,6 +28,32 @@ dependencies = [
  "term",
 ]
 
+[[package]]
+name = "addr2line"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
+[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom",
+ "once_cell",
+ "version_check",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.0.2"
@@ -36,6 +78,31 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.72"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
+dependencies = [
+ "backtrace",
+]
+
+[[package]]
+name = "ariadne"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "367fd0ad87307588d087544707bc5fbf4805ded96c7db922b70d368fa1cb5702"
+dependencies = [
+ "unicode-width",
+ "yansi",
+]
+
+[[package]]
+name = "arrayref"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545"
+
 [[package]]
 name = "arrayvec"
 version = "0.7.4"
@@ -48,6 +115,21 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "backtrace"
+version = "0.3.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "beef"
 version = "0.5.2"
@@ -60,6 +142,29 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "blake3"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5"
+dependencies = [
+ "arrayref",
+ "arrayvec",
+ "cc",
+ "cfg-if",
+ "constant_time_eq",
+ "digest",
+]
+
+[[package]]
+name = "block-buffer"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.13.0"
@@ -93,6 +198,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "chumsky"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23170228b96236b5a7299057ac284a321457700bc8c41a4476052f0f4ba5349d"
+dependencies = [
+ "hashbrown 0.12.3",
+ "stacker",
+]
+
 [[package]]
 name = "codespan-reporting"
 version = "0.11.1"
@@ -103,6 +218,12 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "constant_time_eq"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
+
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.4"
@@ -177,10 +298,41 @@ dependencies = [
 ]
 
 [[package]]
-name = "cxx"
-version = "1.0.101"
+name = "crypto-common"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5032837c1384de3708043de9d4e97bb91290faca6c16529a28aa340592a78166"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
+[[package]]
+name = "csv"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "cxx"
+version = "1.0.102"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f68e12e817cb19eaab81aaec582b4052d07debd3c3c6b083b9d361db47c7dc9d"
 dependencies = [
  "cc",
  "cxxbridge-flags",
@@ -190,9 +342,9 @@ dependencies = [
 
 [[package]]
 name = "cxx-build"
-version = "1.0.101"
+version = "1.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51368b3d0dbf356e10fcbfd455a038503a105ee556f7ee79b6bb8c53a7247456"
+checksum = "e789217e4ab7cf8cc9ce82253180a9fe331f35f5d339f0ccfe0270b39433f397"
 dependencies = [
  "cc",
  "codespan-reporting",
@@ -200,24 +352,24 @@ dependencies = [
  "proc-macro2",
  "quote",
  "scratch",
- "syn 2.0.26",
+ "syn 2.0.27",
 ]
 
 [[package]]
 name = "cxxbridge-flags"
-version = "1.0.101"
+version = "1.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d9062157072e4aafc8e56ceaf8325ce850c5ae37578c852a0d4de2cecdded13"
+checksum = "78a19f4c80fd9ab6c882286fa865e92e07688f4387370a209508014ead8751d0"
 
 [[package]]
 name = "cxxbridge-macro"
-version = "1.0.101"
+version = "1.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf01e8a540f5a4e0f284595834f81cf88572f244b768f051724537afa99a2545"
+checksum = "b8fcfa71f66c8563c4fa9dd2bb68368d50267856f831ac5d85367e0805f9606c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.26",
+ "syn 2.0.27",
 ]
 
 [[package]]
@@ -296,6 +448,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+ "subtle",
+]
+
 [[package]]
 name = "dirs-next"
 version = "2.0.0"
@@ -319,9 +482,27 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.8.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
+checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+
+[[package]]
+name = "enum-as-inner"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
 [[package]]
 name = "fnv"
@@ -338,6 +519,16 @@ dependencies = [
  "thread_local",
 ]
 
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.10"
@@ -349,6 +540,33 @@ dependencies = [
  "wasi 0.11.0+wasi-snapshot-preview1",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.27.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "ahash",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
 [[package]]
 name = "hermit-abi"
 version = "0.3.2"
@@ -384,6 +602,31 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
+[[package]]
+name = "indexmap"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.0",
+]
+
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
+
 [[package]]
 name = "js-sys"
 version = "0.3.64"
@@ -444,6 +687,21 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
+dependencies = [
+ "adler",
+]
+
 [[package]]
 name = "nix"
 version = "0.24.3"
@@ -470,10 +728,20 @@ dependencies = [
 ]
 
 [[package]]
-name = "num-traits"
-version = "0.2.15"
+name = "nom"
+version = "7.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
 dependencies = [
  "autocfg",
 ]
@@ -488,6 +756,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "object"
+version = "0.31.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.18.0"
@@ -509,6 +786,41 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prql-compiler"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c99b52154002ac7f286dd2293c2f8d4e30526c1d396b14deef5ada1deef3c9ff"
+dependencies = [
+ "anyhow",
+ "ariadne",
+ "chumsky",
+ "csv",
+ "enum-as-inner",
+ "itertools",
+ "lazy_static",
+ "log",
+ "once_cell",
+ "regex",
+ "semver",
+ "serde",
+ "serde_json",
+ "serde_yaml",
+ "sqlformat",
+ "sqlparser",
+ "strum",
+ "strum_macros",
+]
+
+[[package]]
+name = "psm"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.31"
@@ -589,12 +901,24 @@ version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
+
 [[package]]
 name = "rustversion"
 version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
 
+[[package]]
+name = "ryu"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -608,10 +932,57 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152"
 
 [[package]]
-name = "serde"
-version = "1.0.171"
+name = "semver"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9"
+checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.174"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b88756493a5bd5e5395d53baa70b194b05764ab85b59e43e4b8f4e1192fa9b1"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.174"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e5c3a298c7f978e53536f95a63bdc4c4a64550582f31a0359a9afda6aede62e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.27",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_yaml"
+version = "0.9.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
 
 [[package]]
 name = "skim"
@@ -638,12 +1009,74 @@ dependencies = [
  "vte",
 ]
 
+[[package]]
+name = "sqlformat"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e"
+dependencies = [
+ "itertools",
+ "nom",
+ "unicode_categories",
+]
+
+[[package]]
+name = "sqlparser"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "355dc4d4b6207ca8a3434fc587db0a8016130a574dbcdbfb93d7f7b5bc5b211a"
+dependencies = [
+ "log",
+ "serde",
+]
+
+[[package]]
+name = "stacker"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "libc",
+ "psm",
+ "winapi",
+]
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
+[[package]]
+name = "strum"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
+dependencies = [
+ "strum_macros",
+]
+
+[[package]]
+name = "strum_macros"
+version = "0.24.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "subtle"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
+
 [[package]]
 name = "syn"
 version = "1.0.109"
@@ -657,9 +1090,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.26"
+version = "2.0.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970"
+checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -688,22 +1121,22 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.43"
+version = "1.0.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42"
+checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.43"
+version = "1.0.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f"
+checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.26",
+ "syn 2.0.27",
 ]
 
 [[package]]
@@ -766,6 +1199,12 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "typenum"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.11"
@@ -778,12 +1217,30 @@ version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
 
+[[package]]
+name = "unicode_categories"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
+
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa"
+
 [[package]]
 name = "utf8parse"
 version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
 [[package]]
 name = "vte"
 version = "0.11.1"
@@ -838,7 +1295,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.26",
+ "syn 2.0.27",
  "wasm-bindgen-shared",
 ]
 
@@ -860,7 +1317,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.26",
+ "syn 2.0.27",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -967,3 +1424,9 @@ name = "windows_x86_64_msvc"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+
+[[package]]
+name = "yansi"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"

rust/Cargo.toml

@@ -0,0 +1,12 @@
+# workspace is required to vendor crates for all packages.
+[workspace]
+members = [
+    "BLAKE3",
+    "skim",
+    "prql",
+]
+resolver = "2"
+
+# FIXME: even though the profiles should be defined in the main cargo config we
+# cannot do this yet, since we compile each package separatelly, so you should
+# ignore warning from cargo about this.

rust/prql/Cargo.lock

@@ -1,569 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "_ch_rust_prql"
-version = "0.1.0"
-dependencies = [
- "prql-compiler",
- "serde_json",
-]
-
-[[package]]
-name = "addr2line"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
-dependencies = [
- "gimli",
-]
-
-[[package]]
-name = "adler"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
-
-[[package]]
-name = "ahash"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
-dependencies = [
- "getrandom",
- "once_cell",
- "version_check",
-]
-
-[[package]]
-name = "aho-corasick"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "anyhow"
-version = "1.0.71"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
-dependencies = [
- "backtrace",
-]
-
-[[package]]
-name = "ariadne"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "367fd0ad87307588d087544707bc5fbf4805ded96c7db922b70d368fa1cb5702"
-dependencies = [
- "unicode-width",
- "yansi",
-]
-
-[[package]]
-name = "backtrace"
-version = "0.3.68"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
-dependencies = [
- "addr2line",
- "cc",
- "cfg-if",
- "libc",
- "miniz_oxide",
- "object",
- "rustc-demangle",
-]
-
-[[package]]
-name = "cc"
-version = "1.0.79"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "chumsky"
-version = "0.9.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23170228b96236b5a7299057ac284a321457700bc8c41a4476052f0f4ba5349d"
-dependencies = [
- "hashbrown 0.12.3",
- "stacker",
-]
-
-[[package]]
-name = "csv"
-version = "1.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086"
-dependencies = [
- "csv-core",
- "itoa",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "csv-core"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "either"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
-
-[[package]]
-name = "enum-as-inner"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116"
-dependencies = [
- "heck",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "equivalent"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1"
-
-[[package]]
-name = "getrandom"
-version = "0.2.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi",
-]
-
-[[package]]
-name = "gimli"
-version = "0.27.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
-
-[[package]]
-name = "hashbrown"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
-dependencies = [
- "ahash",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
-
-[[package]]
-name = "heck"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
-
-[[package]]
-name = "indexmap"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
-dependencies = [
- "equivalent",
- "hashbrown 0.14.0",
-]
-
-[[package]]
-name = "itertools"
-version = "0.10.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
-dependencies = [
- "either",
-]
-
-[[package]]
-name = "itoa"
-version = "1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a"
-
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-
-[[package]]
-name = "libc"
-version = "0.2.147"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
-
-[[package]]
-name = "log"
-version = "0.4.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
-
-[[package]]
-name = "memchr"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
-
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
-[[package]]
-name = "miniz_oxide"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
-dependencies = [
- "adler",
-]
-
-[[package]]
-name = "nom"
-version = "7.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
-dependencies = [
- "memchr",
- "minimal-lexical",
-]
-
-[[package]]
-name = "object"
-version = "0.31.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.63"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "prql-compiler"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c99b52154002ac7f286dd2293c2f8d4e30526c1d396b14deef5ada1deef3c9ff"
-dependencies = [
- "anyhow",
- "ariadne",
- "chumsky",
- "csv",
- "enum-as-inner",
- "itertools",
- "lazy_static",
- "log",
- "once_cell",
- "regex",
- "semver",
- "serde",
- "serde_json",
- "serde_yaml",
- "sqlformat",
- "sqlparser",
- "strum",
- "strum_macros",
-]
-
-[[package]]
-name = "psm"
-version = "0.1.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874"
-dependencies = [
- "cc",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.29"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "regex"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89089e897c013b3deb627116ae56a6955a72b8bed395c9526af31c9fe528b484"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa250384981ea14565685dea16a9ccc4d1c541a13f82b9c168572264d1df8c56"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab07dc67230e4a4718e70fd5c20055a4334b121f1f9db8fe63ef39ce9b8c846"
-
-[[package]]
-name = "rustc-demangle"
-version = "0.1.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
-
-[[package]]
-name = "rustversion"
-version = "1.0.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc31bd9b61a32c31f9650d18add92aa83a49ba979c143eefd27fe7177b05bd5f"
-
-[[package]]
-name = "ryu"
-version = "1.0.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9"
-
-[[package]]
-name = "semver"
-version = "1.0.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "serde"
-version = "1.0.166"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d01b7404f9d441d3ad40e6a636a7782c377d2abdbe4fa2440e2edcc2f4f10db8"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.166"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dd83d6dde2b6b2d466e14d9d1acce8816dedee94f735eac6395808b3483c6d6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c"
-dependencies = [
- "itoa",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "serde_yaml"
-version = "0.9.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85"
-dependencies = [
- "indexmap",
- "itoa",
- "ryu",
- "serde",
- "unsafe-libyaml",
-]
-
-[[package]]
-name = "sqlformat"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e"
-dependencies = [
- "itertools",
- "nom",
- "unicode_categories",
-]
-
-[[package]]
-name = "sqlparser"
-version = "0.33.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "355dc4d4b6207ca8a3434fc587db0a8016130a574dbcdbfb93d7f7b5bc5b211a"
-dependencies = [
- "log",
- "serde",
-]
-
-[[package]]
-name = "stacker"
-version = "0.1.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce"
-dependencies = [
- "cc",
- "cfg-if",
- "libc",
- "psm",
- "winapi",
-]
-
-[[package]]
-name = "strum"
-version = "0.24.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
-dependencies = [
- "strum_macros",
-]
-
-[[package]]
-name = "strum_macros"
-version = "0.24.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
-dependencies = [
- "heck",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "syn"
-version = "1.0.109"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73"
-
-[[package]]
-name = "unicode-width"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
-
-[[package]]
-name = "unicode_categories"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
-
-[[package]]
-name = "unsafe-libyaml"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6"
-
-[[package]]
-name = "version_check"
-version = "0.9.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
-
-[[package]]
-name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
-[[package]]
-name = "yansi"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"

src/Common/CurrentMetrics.cpp

@@ -187,6 +187,7 @@
     M(CacheFileSegments, "Number of existing cache file segments") \
     M(CacheDetachedFileSegments, "Number of existing detached cache file segments") \
     M(FilesystemCacheSize, "Filesystem cache size in bytes") \
+    M(FilesystemCacheSizeLimit, "Filesystem cache size limit in bytes") \
     M(FilesystemCacheElements, "Filesystem cache elements (file segments)") \
     M(FilesystemCacheDownloadQueueElements, "Filesystem cache elements in download queue") \
     M(AsyncInsertCacheSize, "Number of async insert hash id in cache") \

src/Common/MemoryTracker.cpp

@@ -229,7 +229,7 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT
     }
 
     std::bernoulli_distribution sample(sample_probability);
-    if (unlikely(sample_probability > 0.0 && isSizeOkForSampling(size) && sample(thread_local_rng)))
+    if (unlikely(sample_probability > 0.0 && sample(thread_local_rng)))
     {
         MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
         DB::TraceSender::send(DB::TraceType::MemorySample, StackTrace(), {.size = size});
@@ -413,7 +413,7 @@ void MemoryTracker::free(Int64 size)
     }
 
     std::bernoulli_distribution sample(sample_probability);
-    if (unlikely(sample_probability > 0.0 && isSizeOkForSampling(size) && sample(thread_local_rng)))
+    if (unlikely(sample_probability > 0.0 && sample(thread_local_rng)))
     {
         MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
         DB::TraceSender::send(DB::TraceType::MemorySample, StackTrace(), {.size = -size});
@@ -534,12 +534,6 @@ void MemoryTracker::setOrRaiseProfilerLimit(Int64 value)
         ;
 }
 
-bool MemoryTracker::isSizeOkForSampling(UInt64 size) const
-{
-    /// We can avoid comparison min_allocation_size_bytes with zero, because we cannot have 0 bytes allocation/deallocation
-    return ((max_allocation_size_bytes == 0 || size <= max_allocation_size_bytes) && size >= min_allocation_size_bytes);
-}
-
 bool canEnqueueBackgroundTask()
 {
     auto limit = background_memory_tracker.getSoftLimit();

src/Common/MemoryTracker.h

@@ -67,12 +67,6 @@ private:
     /// To randomly sample allocations and deallocations in trace_log.
     double sample_probability = 0;
 
-    /// Randomly sample allocations only larger or equal to this size
-    UInt64 min_allocation_size_bytes = 0;
-
-    /// Randomly sample allocations only smaller or equal to this size
-    UInt64 max_allocation_size_bytes = 0;
-
     /// Singly-linked list. All information will be passed to subsequent memory trackers also (it allows to implement trackers hierarchy).
     /// In terms of tree nodes it is the list of parents. Lifetime of these trackers should "include" lifetime of current tracker.
     std::atomic<MemoryTracker *> parent {};
@@ -94,8 +88,6 @@ private:
 
     void setOrRaiseProfilerLimit(Int64 value);
 
-    bool isSizeOkForSampling(UInt64 size) const;
-
     /// allocImpl(...) and free(...) should not be used directly
     friend struct CurrentMemoryTracker;
     void allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryTracker * query_tracker = nullptr);
@@ -173,16 +165,6 @@ public:
         sample_probability = value;
     }
 
-    void setSampleMinAllocationSize(UInt64 value)
-    {
-        min_allocation_size_bytes = value;
-    }
-
-    void setSampleMaxAllocationSize(UInt64 value)
-    {
-        max_allocation_size_bytes = value;
-    }
-
     void setProfilerStep(Int64 value)
     {
         profiler_step = value;

src/Common/OptimizedRegularExpression.cpp

@@ -1,5 +1,6 @@
 #include <limits>
 #include <Common/Exception.h>
+#include <Common/logger_useful.h>
 #include <Common/PODArray.h>
 #include <Common/checkStackSize.h>
 #include <Common/OptimizedRegularExpression.h>
@@ -423,6 +424,7 @@ void OptimizedRegularExpressionImpl<thread_safe>::analyze(
         bool & is_trivial,
         bool & required_substring_is_prefix,
         std::vector<std::string> & alternatives)
+try
 {
     Literals alternative_literals;
     Literal required_literal;
@@ -432,12 +434,20 @@ void OptimizedRegularExpressionImpl<thread_safe>::analyze(
     for (auto & lit : alternative_literals)
         alternatives.push_back(std::move(lit.literal));
 }
+catch (...)
+{
+    required_substring = "";
+    is_trivial = false;
+    required_substring_is_prefix = false;
+    alternatives.clear();
+    LOG_ERROR(&Poco::Logger::get("OptimizeRegularExpression"), "Analyze RegularExpression failed, got error: {}", DB::getCurrentExceptionMessage(false));
+}
 
 template <bool thread_safe>
 OptimizedRegularExpressionImpl<thread_safe>::OptimizedRegularExpressionImpl(const std::string & regexp_, int options)
 {
-    std::vector<std::string> alternativesDummy; /// this vector extracts patterns a,b,c from pattern (a|b|c). for now it's not used.
-    analyze(regexp_, required_substring, is_trivial, required_substring_is_prefix, alternativesDummy);
+    std::vector<std::string> alternatives_dummy; /// this vector extracts patterns a,b,c from pattern (a|b|c). for now it's not used.
+    analyze(regexp_, required_substring, is_trivial, required_substring_is_prefix, alternatives_dummy);
 
 
     /// Just three following options are supported

src/Common/ZooKeeper/ZooKeeper.cpp

@@ -15,6 +15,7 @@
 #include <base/sort.h>
 #include <base/getFQDNOrHostName.h>
 #include "Common/ZooKeeper/IKeeper.h"
+#include <Common/DNSResolver.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/Exception.h>
 #include <Common/logger_useful.h>
@@ -82,6 +83,9 @@ void ZooKeeper::init(ZooKeeperArgs args_)
                 if (secure)
                     host_string.erase(0, strlen("secure://"));
 
+                /// We want to resolve all hosts without DNS cache for keeper connection.
+                Coordination::DNSResolver::instance().removeHostFromCache(host_string);
+
                 const Poco::Net::SocketAddress host_socket_addr{host_string};
                 LOG_TEST(log, "Adding ZooKeeper host {} ({})", host_string, host_socket_addr.toString());
                 nodes.emplace_back(Coordination::ZooKeeper::Node{host_socket_addr, secure});

src/Core/MySQL/IMySQLReadPacket.cpp

@@ -43,11 +43,12 @@ void LimitedReadPacket::readPayloadWithUnpacked(ReadBuffer & in)
     IMySQLReadPacket::readPayloadWithUnpacked(limited);
 }
 
-uint64_t readLengthEncodedNumber(ReadBuffer & buffer)
+uint64_t readLengthEncodedNumber(ReadBuffer & buffer, UInt16 & bytes_read)
 {
     char c{};
     uint64_t buf = 0;
     buffer.readStrict(c);
+    bytes_read = 1;
     auto cc = static_cast<uint8_t>(c);
     switch (cc)
     {
@@ -56,12 +57,15 @@ uint64_t readLengthEncodedNumber(ReadBuffer & buffer)
             break;
         case 0xfc:
             buffer.readStrict(reinterpret_cast<char *>(&buf), 2);
+            bytes_read += 2;
             break;
         case 0xfd:
             buffer.readStrict(reinterpret_cast<char *>(&buf), 3);
+            bytes_read += 3;
             break;
         case 0xfe:
             buffer.readStrict(reinterpret_cast<char *>(&buf), 8);
+            bytes_read += 8;
             break;
         default:
             return cc;
@@ -69,6 +73,12 @@ uint64_t readLengthEncodedNumber(ReadBuffer & buffer)
     return buf;
 }
 
+uint64_t readLengthEncodedNumber(ReadBuffer & buffer)
+{
+    UInt16 bytes_read = 0;
+    return readLengthEncodedNumber(buffer, bytes_read);
+}
+
 void readLengthEncodedString(String & s, ReadBuffer & buffer)
 {
     uint64_t len = readLengthEncodedNumber(buffer);

src/Core/MySQL/IMySQLReadPacket.h

@@ -34,6 +34,7 @@ public:
 };
 
 uint64_t readLengthEncodedNumber(ReadBuffer & buffer);
+uint64_t readLengthEncodedNumber(ReadBuffer & buffer, UInt16 & bytes_read);
 void readLengthEncodedString(String & s, ReadBuffer & buffer);
 
 }

src/Core/MySQL/MySQLCharset.cpp

@@ -0,0 +1,301 @@
+#include "MySQLCharset.h"
+#include "config.h"
+#include <iostream>
+#include <Common/Exception.h>
+
+#if USE_ICU
+#include <unicode/ucnv.h>
+#define CHUNK_SIZE 1024
+static const char * TARGET_CHARSET = "utf8";
+#endif
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int UNKNOWN_EXCEPTION;
+}
+
+const std::unordered_map<Int32, String> MySQLCharset::charsets
+    = {
+          {1, "big5"},
+          {2, "latin2"},
+          {3, "dec8"},
+          {4, "cp850"},
+          {5, "latin1"},
+          {6, "hp8"},
+          {7, "koi8r"},
+          {8, "latin1"},
+          {9, "latin2"},
+          {10, "swe7"},
+          {11, "ascii"},
+          {12, "ujis"},
+          {13, "sjis"},
+          {14, "cp1251"},
+          {15, "latin1"},
+          {16, "hebrew"},
+          {18, "tis620"},
+          {19, "euckr"},
+          {20, "latin7"},
+          {21, "latin2"},
+          {22, "koi8u"},
+          {23, "cp1251"},
+          {24, "gb2312"},
+          {25, "greek"},
+          {26, "cp1250"},
+          {27, "latin2"},
+          {28, "gbk"},
+          {29, "cp1257"},
+          {30, "latin5"},
+          {31, "latin1"},
+          {32, "armscii8"},
+          {34, "cp1250"},
+          {35, "ucs2"},
+          {36, "cp866"},
+          {37, "keybcs2"},
+          {38, "macce"},
+          {39, "macroman"},
+          {40, "cp852"},
+          {41, "latin7"},
+          {42, "latin7"},
+          {43, "macce"},
+          {44, "cp1250"},
+          {47, "latin1"},
+          {48, "latin1"},
+          {49, "latin1"},
+          {50, "cp1251"},
+          {51, "cp1251"},
+          {52, "cp1251"},
+          {53, "macroman"},
+          {54, "utf16"},
+          {55, "utf16"},
+          {56, "utf16le"},
+          {57, "cp1256"},
+          {58, "cp1257"},
+          {59, "cp1257"},
+          {60, "utf32"},
+          {61, "utf32"},
+          {62, "utf16le"},
+          {64, "armscii8"},
+          {65, "ascii"},
+          {66, "cp1250"},
+          {67, "cp1256"},
+          {68, "cp866"},
+          {69, "dec8"},
+          {70, "greek"},
+          {71, "hebrew"},
+          {72, "hp8"},
+          {73, "keybcs2"},
+          {74, "koi8r"},
+          {75, "koi8u"},
+          {77, "latin2"},
+          {78, "latin5"},
+          {79, "latin7"},
+          {80, "cp850"},
+          {81, "cp852"},
+          {82, "swe7"},
+          {84, "big5"},
+          {85, "euckr"},
+          {86, "gb2312"},
+          {87, "gbk"},
+          {88, "sjis"},
+          {89, "tis620"},
+          {90, "ucs2"},
+          {91, "ujis"},
+          {92, "geostd8"},
+          {93, "geostd8"},
+          {94, "latin1"},
+          {95, "cp932"},
+          {96, "cp932"},
+          {97, "eucjpms"},
+          {98, "eucjpms"},
+          {99, "cp1250"},
+          {101, "utf16"},
+          {102, "utf16"},
+          {103, "utf16"},
+          {104, "utf16"},
+          {105, "utf16"},
+          {106, "utf16"},
+          {107, "utf16"},
+          {108, "utf16"},
+          {109, "utf16"},
+          {110, "utf16"},
+          {111, "utf16"},
+          {112, "utf16"},
+          {113, "utf16"},
+          {114, "utf16"},
+          {115, "utf16"},
+          {116, "utf16"},
+          {117, "utf16"},
+          {118, "utf16"},
+          {119, "utf16"},
+          {120, "utf16"},
+          {121, "utf16"},
+          {122, "utf16"},
+          {123, "utf16"},
+          {124, "utf16"},
+          {128, "ucs2"},
+          {129, "ucs2"},
+          {130, "ucs2"},
+          {131, "ucs2"},
+          {132, "ucs2"},
+          {133, "ucs2"},
+          {134, "ucs2"},
+          {135, "ucs2"},
+          {136, "ucs2"},
+          {137, "ucs2"},
+          {138, "ucs2"},
+          {139, "ucs2"},
+          {140, "ucs2"},
+          {141, "ucs2"},
+          {142, "ucs2"},
+          {143, "ucs2"},
+          {144, "ucs2"},
+          {145, "ucs2"},
+          {146, "ucs2"},
+          {147, "ucs2"},
+          {148, "ucs2"},
+          {149, "ucs2"},
+          {150, "ucs2"},
+          {151, "ucs2"},
+          {159, "ucs2"},
+          {160, "utf32"},
+          {161, "utf32"},
+          {162, "utf32"},
+          {163, "utf32"},
+          {164, "utf32"},
+          {165, "utf32"},
+          {166, "utf32"},
+          {167, "utf32"},
+          {168, "utf32"},
+          {169, "utf32"},
+          {170, "utf32"},
+          {171, "utf32"},
+          {172, "utf32"},
+          {173, "utf32"},
+          {174, "utf32"},
+          {175, "utf32"},
+          {176, "utf32"},
+          {177, "utf32"},
+          {178, "utf32"},
+          {179, "utf32"},
+          {180, "utf32"},
+          {181, "utf32"},
+          {182, "utf32"},
+          {183, "utf32"},
+          {248, "gb18030"},
+          {249, "gb18030"},
+          {250, "gb18030"}
+      };
+
+MySQLCharset::~MySQLCharset()
+{
+#if USE_ICU
+    std::lock_guard lock(mutex);
+    for (auto & conv : conv_cache)
+    {
+        ucnv_close(conv.second);
+    }
+    conv_cache.clear();
+#endif
+}
+
+bool MySQLCharset::needConvert(UInt32 id)
+{
+    return charsets.contains(id);
+}
+
+String MySQLCharset::getCharsetFromId(UInt32 id)
+{
+    return charsets.at(id);
+}
+
+UConverter * MySQLCharset::getCachedConverter(const String & charset [[maybe_unused]])
+{
+    UConverter * conv = nullptr;
+#if USE_ICU
+    UErrorCode error = U_ZERO_ERROR;
+    /// Get conv from cache
+    auto result = conv_cache.find(charset);
+    if (result != conv_cache.end())
+    {
+        conv = result->second;
+        //reset to init state
+        ucnv_reset(conv);
+    }
+    else
+    {
+        conv = ucnv_open(charset.c_str(), &error);
+        if (error != U_ZERO_ERROR)
+        {
+            throw Exception(
+                ErrorCodes::UNKNOWN_EXCEPTION, "MySQLCharset::getCachedConveter: ucnv_open failed, error={}", std::to_string(error));
+        }
+        conv_cache[charset.c_str()] = conv;
+    }
+#endif
+    return conv;
+}
+
+Int32 MySQLCharset::convertFromId(UInt32 id [[maybe_unused]], String & to, const String & from)
+{
+#if USE_ICU
+    std::lock_guard lock(mutex);
+    UErrorCode error = U_ZERO_ERROR;
+    String source_charset = getCharsetFromId(id);
+    to.clear();
+    if (source_charset.empty())
+    {
+        return U_ILLEGAL_ARGUMENT_ERROR;
+    }
+
+    UChar pivot_buf[CHUNK_SIZE]; // stream mode must use this buf
+    char target_buf[CHUNK_SIZE];
+    UChar * pivot;
+    UChar * pivot2;
+    UConverter * in_conv;
+    UConverter * out_conv;
+    char * cur_target;
+    const char * source_end;
+    const char * target_end;
+
+    size_t source_len = from.size();
+    const char * source = from.data();
+    source_end = source + source_len;
+
+    out_conv = getCachedConverter(TARGET_CHARSET);
+    in_conv = getCachedConverter(source_charset);
+    pivot = pivot_buf;
+    pivot2 = pivot_buf;
+
+    target_end = target_buf + CHUNK_SIZE;
+    do
+    {
+        error = U_ZERO_ERROR;
+        cur_target = target_buf;
+        ucnv_convertEx(
+            out_conv,
+            in_conv,
+            &cur_target,
+            target_end,
+            &source,
+            source_end,
+            pivot_buf,
+            &pivot,
+            &pivot2,
+            pivot_buf + CHUNK_SIZE,
+            false,
+            true,
+            &error);
+        to.append(target_buf, cur_target - target_buf);
+    } while (error == U_BUFFER_OVERFLOW_ERROR);
+
+    return error;
+#else
+    to = from;
+    return 0;
+#endif
+}
+
+}

src/Core/MySQL/MySQLCharset.h

@@ -0,0 +1,26 @@
+#pragma once
+#include <unordered_map>
+#include <base/types.h>
+#include <boost/noncopyable.hpp>
+#include <mutex>
+
+struct UConverter;
+
+namespace DB
+{
+class MySQLCharset final : boost::noncopyable
+{
+public:
+    ~MySQLCharset();
+    String getCharsetFromId(UInt32 id);
+    Int32 convertFromId(UInt32 id, String & to, const String & from);
+    bool needConvert(UInt32 id);
+private:
+    std::mutex mutex;
+    std::unordered_map<String, UConverter *> conv_cache;
+    UConverter * getCachedConverter(const String & charset);
+    static const std::unordered_map<Int32, String> charsets;
+};
+
+using MySQLCharsetPtr = std::shared_ptr<MySQLCharset>;
+}

src/Core/MySQL/MySQLReplication.cpp

@@ -187,9 +187,9 @@ namespace MySQLReplication
         size_t null_bitmap_size = (column_count + 7) / 8;
         readBitmap(payload, null_bitmap, null_bitmap_size);
 
-        /// Ignore MySQL 8.0 optional metadata fields.
+        /// Parse MySQL 8.0 optional metadata fields.
         /// https://mysqlhighavailability.com/more-metadata-is-written-into-binary-log/
-        payload.ignoreAll();
+        parseOptionalMetaField(payload);
     }
 
     /// Types that do not used in the binlog event:
@@ -263,6 +263,118 @@ namespace MySQLReplication
         }
     }
 
+    void TableMapEvent::parseOptionalMetaField(ReadBuffer & payload)
+    {
+        char type = 0;
+        while (payload.read(type))
+        {
+            UInt64 len = readLengthEncodedNumber(payload);
+            if (len == 0)
+            {
+                payload.ignoreAll();
+                return;
+            }
+            switch (type)
+            {
+                /// It may be useful, parse later
+                case SIGNEDNESS:
+                    payload.ignore(len);
+                    break;
+                case DEFAULT_CHARSET:
+                {
+                    UInt32 total_read = 0;
+                    UInt16 once_read = 0;
+                    default_charset = static_cast<UInt32>(readLengthEncodedNumber(payload, once_read));
+                    total_read += once_read;
+                    while (total_read < len)
+                    {
+                        UInt32 col_index = static_cast<UInt32>(readLengthEncodedNumber(payload, once_read));
+                        total_read += once_read;
+                        UInt32 col_charset = static_cast<UInt32>(readLengthEncodedNumber(payload, once_read));
+                        total_read += once_read;
+                        default_charset_pairs.emplace(col_index, col_charset);
+                    }
+                    break;
+                }
+                case COLUMN_CHARSET:
+                {
+                    UInt32 total_read = 0;
+                    UInt16 once_read = 0;
+                    while (total_read < len)
+                    {
+                        UInt32 collation_id = static_cast<UInt32>(readLengthEncodedNumber(payload, once_read));
+                        column_charset.emplace_back(collation_id);
+                        total_read += once_read;
+                    }
+                    break;
+                }
+                case COLUMN_NAME:
+                    payload.ignore(len);
+                    break;
+                case SET_STR_VALUE:
+                case GEOMETRY_TYPE:
+                case SIMPLE_PRIMARY_KEY:
+                case PRIMARY_KEY_WITH_PREFIX:
+                case ENUM_AND_SET_DEFAULT_CHARSET:
+                case COLUMN_VISIBILITY:
+                default:
+                    payload.ignore(len);
+                    break;
+            }
+        }
+    }
+
+    UInt32 TableMapEvent::getColumnCharsetId(UInt32 column_index)
+    {
+        if (!column_charset.empty())
+        {
+            UInt32 str_index = 0xFFFFFFFF;
+            /// Calc the index in the column_charset
+            for (UInt32 i = 0; i <= column_index; ++i)
+            {
+                switch (column_type[i])
+                {
+                    case MYSQL_TYPE_STRING:
+                    case MYSQL_TYPE_VAR_STRING:
+                    case MYSQL_TYPE_VARCHAR:
+                    case MYSQL_TYPE_BLOB:
+                        ++str_index;
+                        break;
+                    default:
+                        break;
+                }
+            }
+
+            if (str_index != 0xFFFFFFFF && str_index < column_charset.size())
+            {
+                return column_charset[str_index];
+            }
+        }
+        else if (!default_charset_pairs.empty())
+        {
+            UInt32 str_index = 0xFFFFFFFF;
+            for (UInt32 i = 0; i <= column_index; ++i)
+            {
+                switch (column_type[i])
+                {
+                    case MYSQL_TYPE_STRING:
+                    case MYSQL_TYPE_VAR_STRING:
+                    case MYSQL_TYPE_VARCHAR:
+                    case MYSQL_TYPE_BLOB:
+                        ++str_index;
+                        break;
+                    default:
+                        break;
+                }
+            }
+            if (default_charset_pairs.contains(str_index))
+            {
+                return default_charset_pairs[str_index];
+            }
+        }
+        return default_charset;
+    }
+
     void TableMapEvent::dump(WriteBuffer & out) const
     {
         header.dump(out);
@@ -319,6 +431,22 @@ namespace MySQLReplication
         }
     }
 
+    static inline String convertCharsetIfNeeded(
+        const std::shared_ptr<TableMapEvent> & table_map,
+        UInt32 i,
+        const String & val)
+    {
+        const auto collation_id = table_map->getColumnCharsetId(i);
+        if (table_map->charset_ptr->needConvert(collation_id))
+        {
+            String target;
+            auto err = table_map->charset_ptr->convertFromId(collation_id, target, val);
+            if (err == 0)
+                return target;
+        }
+        return val;
+    }
+
     /// Types that do not used in the binlog event:
     /// MYSQL_TYPE_SET
     /// MYSQL_TYPE_TINY_BLOB
@@ -727,7 +855,7 @@ namespace MySQLReplication
                         String val;
                         val.resize(size);
                         payload.readStrict(reinterpret_cast<char *>(val.data()), size);
-                        row.push_back(Field{String{val}});
+                        row.emplace_back(Field{convertCharsetIfNeeded(table_map, i, val)});
                         break;
                     }
                     case MYSQL_TYPE_STRING:
@@ -745,7 +873,7 @@ namespace MySQLReplication
                         String val;
                         val.resize(size);
                         payload.readStrict(reinterpret_cast<char *>(val.data()), size);
-                        row.push_back(Field{String{val}});
+                        row.emplace_back(Field{convertCharsetIfNeeded(table_map, i, val)});
                         break;
                     }
                     case MYSQL_TYPE_GEOMETRY:
@@ -777,7 +905,10 @@ namespace MySQLReplication
                         String val;
                         val.resize(size);
                         payload.readStrict(reinterpret_cast<char *>(val.data()), size);
-                        row.push_back(Field{String{val}});
+                        row.emplace_back(Field{
+                            field_type == MYSQL_TYPE_BLOB
+                            ? convertCharsetIfNeeded(table_map, i, val)
+                            : val});
                         break;
                     }
                     default:
@@ -977,7 +1108,7 @@ namespace MySQLReplication
                 map_event_header.parse(event_payload);
                 if (doReplicate(map_event_header.schema, map_event_header.table))
                 {
-                    event = std::make_shared<TableMapEvent>(std::move(event_header), map_event_header);
+                    event = std::make_shared<TableMapEvent>(std::move(event_header), map_event_header, flavor_charset);
                     event->parseEvent(event_payload);
                     auto table_map = std::static_pointer_cast<TableMapEvent>(event);
                     table_maps[table_map->table_id] = table_map;

src/Core/MySQL/MySQLReplication.h

@@ -2,6 +2,7 @@
 #include <Core/Field.h>
 #include <Core/MySQL/PacketsReplication.h>
 #include <Core/MySQL/MySQLGtid.h>
+#include <Core/MySQL/MySQLCharset.h>
 #include <base/types.h>
 #include <IO/ReadBuffer.h>
 #include <IO/WriteBuffer.h>
@@ -324,9 +325,24 @@ namespace MySQLReplication
         UInt32 column_count;
         std::vector<UInt8> column_type;
         std::vector<UInt16> column_meta;
+        /// Character set of string columns
+        std::vector<UInt32> column_charset;
+        /// Character set of string columns,
+        /// optimized to minimize space when many
+        /// columns have the same charset
+        UInt32 default_charset = 255; /// utf8mb4_0900_ai_ci
+        std::unordered_map<UInt32, UInt32> default_charset_pairs;
+        /// Points to flavor_charset object
+        MySQLCharsetPtr charset_ptr;
         Bitmap null_bitmap;
 
-        TableMapEvent(EventHeader && header_, const TableMapEventHeader & map_event_header) : EventBase(std::move(header_)), column_count(0)
+        TableMapEvent(
+            EventHeader && header_,
+            const TableMapEventHeader & map_event_header,
+            const MySQLCharsetPtr & charset_ptr_)
+            : EventBase(std::move(header_))
+            , column_count(0)
+            , charset_ptr(charset_ptr_)
         {
             table_id = map_event_header.table_id;
             flags = map_event_header.flags;
@@ -336,10 +352,52 @@ namespace MySQLReplication
             table = map_event_header.table;
         }
         void dump(WriteBuffer & out) const override;
+        UInt32 getColumnCharsetId(UInt32 column_index);
+        /// https://mysqlhighavailability.com/more-metadata-is-written-into-binary-log/
+        /// https://github.com/mysql/mysql-server/blob/8.0/libbinlogevents/include/rows_event.h#L50
+        /// DEFAULT_CHARSET and COLUMN_CHARSET don't appear together, and
+        /// ENUM_AND_SET_DEFAULT_CHARSET and ENUM_AND_SET_COLUMN_CHARSET don't appear together.
+        enum OptionalMetaType : char
+        {
+            /// UNSIGNED flag of numeric columns
+            SIGNEDNESS = 1,
+            /// Character set of string columns, optimized to
+            /// minimize space when many columns have the
+            /// same charset
+            DEFAULT_CHARSET,
+            /// Character set of string columns, optimized to
+            /// minimize space when columns have many
+            /// different charsets
+            COLUMN_CHARSET,
+            COLUMN_NAME,
+            /// String value of SET columns
+            SET_STR_VALUE,
+            /// String value of ENUM columns
+            ENUM_STR_VALUE,
+            /// Real type of geometry columns
+            GEOMETRY_TYPE,
+            /// Primary key without prefix
+            SIMPLE_PRIMARY_KEY,
+            /// Primary key with prefix
+            PRIMARY_KEY_WITH_PREFIX,
+            /// Character set of enum and set
+            /// columns, optimized to minimize
+            /// space when many columns have the
+            /// same charset
+            ENUM_AND_SET_DEFAULT_CHARSET,
+            /// Character set of enum and set
+            /// columns, optimized to minimize
+            /// space when many columns have the
+            /// same charset
+            ENUM_AND_SET_COLUMN_CHARSET,
+            /// Flag to indicate column visibility attribute
+            COLUMN_VISIBILITY
+        };
 
     protected:
         void parseImpl(ReadBuffer & payload) override;
         void parseMeta(String meta);
+        void parseOptionalMetaField(ReadBuffer & payload);
     };
 
     enum RowsEventFlags
@@ -486,6 +544,7 @@ namespace MySQLReplication
         std::unordered_set<String> replicate_tables;
         std::map<UInt64, std::shared_ptr<TableMapEvent> > table_maps;
         size_t checksum_signature_length = 4;
+        MySQLCharsetPtr flavor_charset = std::make_shared<MySQLCharset>();
 
         bool doReplicate(UInt64 table_id);
         bool doReplicate(const String & db, const String & table_name);

src/Core/ServerSettings.h

@@ -81,12 +81,8 @@ namespace DB
     M(UInt64, background_schedule_pool_size, 128, "The maximum number of threads that will be used for constantly executing some lightweight periodic operations.", 0) \
     M(UInt64, background_message_broker_schedule_pool_size, 16, "The maximum number of threads that will be used for executing background operations for message streaming.", 0) \
     M(UInt64, background_distributed_schedule_pool_size, 16, "The maximum number of threads that will be used for executing distributed sends.", 0) \
-    M(Bool, display_secrets_in_show_and_select, false, "Allow showing secrets in SHOW and SELECT queries via a format setting and a grant", 0) \
-    \
-    M(UInt64, total_memory_profiler_step, 0, "Whenever server memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down server.", 0) \
-    M(Double, total_memory_tracker_sample_probability, 0, "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation (can be changed with `memory_profiler_sample_min_allocation_size` and `memory_profiler_sample_max_allocation_size`). Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \
-    M(UInt64, total_memory_profiler_sample_min_allocation_size, 0, "Collect random allocations of size greater or equal than specified value with probability equal to `total_memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \
-    M(UInt64, total_memory_profiler_sample_max_allocation_size, 0, "Collect random allocations of size less or equal than specified value with probability equal to `total_memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0)
+    M(Bool, display_secrets_in_show_and_select, false, "Allow showing secrets in SHOW and SELECT queries via a format setting and a grant", 0)
+
 
 DECLARE_SETTINGS_TRAITS(ServerSettingsTraits, SERVER_SETTINGS)
 

src/Core/Settings.h

@@ -427,9 +427,7 @@ class IColumn;
     M(UInt64, memory_overcommit_ratio_denominator_for_user, 1_GiB, "It represents soft memory limit on the global level. This value is used to compute query overcommit ratio.", 0) \
     M(UInt64, max_untracked_memory, (4 * 1024 * 1024), "Small allocations and deallocations are grouped in thread local variable and tracked or profiled only when amount (in absolute value) becomes larger than specified value. If the value is higher than 'memory_profiler_step' it will be effectively lowered to 'memory_profiler_step'.", 0) \
     M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \
-    M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation (can be changed with `memory_profiler_sample_min_allocation_size` and `memory_profiler_sample_max_allocation_size`). Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \
-    M(UInt64, memory_profiler_sample_min_allocation_size, 0, "Collect random allocations of size greater or equal than specified value with probability equal to `memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \
-    M(UInt64, memory_profiler_sample_max_allocation_size, 0, "Collect random allocations of size less or equal than specified value with probability equal to `memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \
+    M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \
     M(Bool, trace_profile_events, false, "Send to system.trace_log profile event and value of increment on each increment with 'ProfileEvent' trace_type", 0) \
     \
     M(UInt64, memory_usage_overcommit_max_wait_microseconds, 5'000'000, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown.", 0) \

src/Core/tests/gtest_charset_conv.cpp

@@ -0,0 +1,351 @@
+#include <Core/MySQL/MySQLCharset.h>
+#include <gtest/gtest.h>
+#include <cstdio>
+
+namespace DB
+{
+
+struct CheckResult
+{
+    Int32 id;
+    String name;
+    bool need_convert;
+};
+
+TEST(CharsetTest, CharsetTest)
+{
+    MySQLCharset charset;
+    UInt32 big5_id = 1;
+    UInt32 gbk_id = 28;
+    UInt32 gb2312_id = 24;
+    UInt32 utf8mb4_ai_ci_id = 255;
+    EXPECT_TRUE(charset.needConvert(big5_id));
+    EXPECT_TRUE(charset.needConvert(gbk_id));
+    EXPECT_TRUE(charset.needConvert(gb2312_id));
+    EXPECT_FALSE(charset.needConvert(utf8mb4_ai_ci_id));
+    EXPECT_FALSE(charset.needConvert(0));
+    EXPECT_FALSE(charset.needConvert(1000));
+
+    EXPECT_EQ(charset.getCharsetFromId(big5_id), String("big5"));
+    EXPECT_EQ(charset.getCharsetFromId(gbk_id), String("gbk"));
+    EXPECT_EQ(charset.getCharsetFromId(gb2312_id), String("gb2312"));
+}
+
+TEST(CharsetTest, ConvTest)
+{
+    MySQLCharset charset;
+    UInt32 big5_id = 1;
+    UInt32 gbk_id = 28;
+    UInt32 gb2312_id = 24;
+    Int32 error = 0;
+    String source("\xc4\xe3\xba\xc3"); // gbk "你好"
+    String target;
+    String expect("\xe4\xbd\xa0\xe5\xa5\xbd");
+
+    error = charset.convertFromId(gbk_id, target, source);
+    EXPECT_EQ(error, 0);
+    EXPECT_TRUE(target == expect);
+
+    error = charset.convertFromId(gb2312_id, target, source);
+    EXPECT_EQ(error, 0);
+    EXPECT_TRUE(target == expect);
+
+    source.assign("\xa7\x41\xa6\x6e"); // big5 "你好"
+    error = charset.convertFromId(big5_id, target, source);
+    EXPECT_EQ(error, 0);
+    EXPECT_TRUE(target == expect);
+}
+
+TEST(CharsetTest, FullCharsetCheck)
+{
+    CheckResult result[] =
+    {
+        {1, "big5", true}, // "big5_chinese_ci",
+        {2, "latin2", true}, // "latin2_czech_cs",
+        {3, "dec8", true}, // "dec8_swedish_ci",
+        {4, "cp850", true}, // "cp850_general_ci",
+        {5, "latin1", true}, // "latin1_german1_ci",
+        {6, "hp8", true}, // "hp8_english_ci",
+        {7, "koi8r", true}, // "koi8r_general_ci",
+        {8, "latin1", true}, // "latin1_swedish_ci",
+        {9, "latin2", true}, // "latin2_general_ci",
+        {10, "swe7", true}, // "swe7_swedish_ci",
+        {11, "ascii", true}, // "ascii_general_ci",
+        {12, "ujis", true}, // "ujis_japanese_ci",
+        {13, "sjis", true}, // "sjis_japanese_ci",
+        {14, "cp1251", true}, // "cp1251_bulgarian_ci",
+        {15, "latin1", true}, // "latin1_danish_ci",
+        {16, "hebrew", true}, // "hebrew_general_ci",
+        {18, "tis620", true}, // "tis620_thai_ci",
+        {19, "euckr", true}, // "euckr_korean_ci",
+        {20, "latin7", true}, // "latin7_estonian_cs",
+        {21, "latin2", true}, // "latin2_hungarian_ci",
+        {22, "koi8u", true}, // "koi8u_general_ci",
+        {23, "cp1251", true}, // "cp1251_ukrainian_ci",
+        {24, "gb2312", true}, // "gb2312_chinese_ci",
+        {25, "greek", true}, // "greek_general_ci",
+        {26, "cp1250", true}, // "cp1250_general_ci",
+        {27, "latin2", true}, // "latin2_croatian_ci",
+        {28, "gbk", true}, // "gbk_chinese_ci",
+        {29, "cp1257", true}, // "cp1257_lithuanian_ci",
+        {30, "latin5", true}, // "latin5_turkish_ci",
+        {31, "latin1", true}, // "latin1_german2_ci",
+        {32, "armscii8", true}, // "armscii8_general_ci",
+        {33, "utf8", false}, // "utf8_general_ci",
+        {34, "cp1250", true}, // "cp1250_czech_cs",
+        {35, "ucs2", true}, // "ucs2_general_ci",
+        {36, "cp866", true}, // "cp866_general_ci",
+        {37, "keybcs2", true}, // "keybcs2_general_ci",
+        {38, "macce", true}, // "macce_general_ci",
+        {39, "macroman", true}, // "macroman_general_ci",
+        {40, "cp852", true}, // "cp852_general_ci",
+        {41, "latin7", true}, // "latin7_general_ci",
+        {42, "latin7", true}, // "latin7_general_cs",
+        {43, "macce", true}, // "macce_bin",
+        {44, "cp1250", true}, // "cp1250_croatian_ci",
+        {45, "utf8mb4", false}, // "utf8mb4_general_ci",
+        {46, "utf8mb4", false}, // "utf8mb4_bin",
+        {47, "latin1", true}, // "latin1_bin",
+        {48, "latin1", true}, // "latin1_general_ci",
+        {49, "latin1", true}, // "latin1_general_cs",
+        {50, "cp1251", true}, // "cp1251_bin",
+        {51, "cp1251", true}, // "cp1251_general_ci",
+        {52, "cp1251", true}, // "cp1251_general_cs",
+        {53, "macroman", true}, // "macroman_bin",
+        {54, "utf16", true}, // "utf16_general_ci",
+        {55, "utf16", true}, // "utf16_bin",
+        {56, "utf16le", true}, // "utf16le_general_ci",
+        {57, "cp1256", true}, // "cp1256_general_ci",
+        {58, "cp1257", true}, // "cp1257_bin",
+        {59, "cp1257", true}, // "cp1257_general_ci",
+        {60, "utf32", true}, // "utf32_general_ci",
+        {61, "utf32", true}, // "utf32_bin",
+        {62, "utf16le", true}, // "utf16le_bin",
+        {64, "armscii8", true}, // "armscii8_bin",
+        {65, "ascii", true}, // "ascii_bin",
+        {66, "cp1250", true}, // "cp1250_bin",
+        {67, "cp1256", true}, // "cp1256_bin",
+        {68, "cp866", true}, // "cp866_bin",
+        {69, "dec8", true}, // "dec8_bin",
+        {70, "greek", true}, // "greek_bin",
+        {71, "hebrew", true}, // "hebrew_bin",
+        {72, "hp8", true}, // "hp8_bin",
+        {73, "keybcs2", true}, // "keybcs2_bin",
+        {74, "koi8r", true}, // "koi8r_bin",
+        {75, "koi8u", true}, // "koi8u_bin",
+        {77, "latin2", true}, // "latin2_bin",
+        {78, "latin5", true}, // "latin5_bin",
+        {79, "latin7", true}, // "latin7_bin",
+        {80, "cp850", true}, // "cp850_bin",
+        {81, "cp852", true}, // "cp852_bin",
+        {82, "swe7", true}, // "swe7_bin",
+        {83, "utf8", false}, // "utf8_bin",
+        {84, "big5", true}, // "big5_bin",
+        {85, "euckr", true}, // "euckr_bin",
+        {86, "gb2312", true}, // "gb2312_bin",
+        {87, "gbk", true}, // "gbk_bin",
+        {88, "sjis", true}, // "sjis_bin",
+        {89, "tis620", true}, // "tis620_bin",
+        {90, "ucs2", true}, // "ucs2_bin",
+        {91, "ujis", true}, // "ujis_bin",
+        {92, "geostd8", true}, // "geostd8_general_ci",
+        {93, "geostd8", true}, // "geostd8_bin",
+        {94, "latin1", true}, // "latin1_spanish_ci",
+        {95, "cp932", true}, // "cp932_japanese_ci",
+        {96, "cp932", true}, // "cp932_bin",
+        {97, "eucjpms", true}, // "eucjpms_japanese_ci",
+        {98, "eucjpms", true}, // "eucjpms_bin",
+        {99, "cp1250", true}, // "cp1250_polish_ci",
+        {101, "utf16", true}, // "utf16_unicode_ci",
+        {102, "utf16", true}, // "utf16_icelandic_ci",
+        {103, "utf16", true}, // "utf16_latvian_ci",
+        {104, "utf16", true}, // "utf16_romanian_ci",
+        {105, "utf16", true}, // "utf16_slovenian_ci",
+        {106, "utf16", true}, // "utf16_polish_ci",
+        {107, "utf16", true}, // "utf16_estonian_ci",
+        {108, "utf16", true}, // "utf16_spanish_ci",
+        {109, "utf16", true}, // "utf16_swedish_ci",
+        {110, "utf16", true}, // "utf16_turkish_ci",
+        {111, "utf16", true}, // "utf16_czech_ci",
+        {112, "utf16", true}, // "utf16_danish_ci",
+        {113, "utf16", true}, // "utf16_lithuanian_ci",
+        {114, "utf16", true}, // "utf16_slovak_ci",
+        {115, "utf16", true}, // "utf16_spanish2_ci",
+        {116, "utf16", true}, // "utf16_roman_ci",
+        {117, "utf16", true}, // "utf16_persian_ci",
+        {118, "utf16", true}, // "utf16_esperanto_ci",
+        {119, "utf16", true}, // "utf16_hungarian_ci",
+        {120, "utf16", true}, // "utf16_sinhala_ci",
+        {121, "utf16", true}, // "utf16_german2_ci",
+        {122, "utf16", true}, // "utf16_croatian_ci",
+        {123, "utf16", true}, // "utf16_unicode_520_ci",
+        {124, "utf16", true}, // "utf16_vietnamese_ci",
+        {128, "ucs2", true}, // "ucs2_unicode_ci",
+        {129, "ucs2", true}, // "ucs2_icelandic_ci",
+        {130, "ucs2", true}, // "ucs2_latvian_ci",
+        {131, "ucs2", true}, // "ucs2_romanian_ci",
+        {132, "ucs2", true}, // "ucs2_slovenian_ci",
+        {133, "ucs2", true}, // "ucs2_polish_ci",
+        {134, "ucs2", true}, // "ucs2_estonian_ci",
+        {135, "ucs2", true}, // "ucs2_spanish_ci",
+        {136, "ucs2", true}, // "ucs2_swedish_ci",
+        {137, "ucs2", true}, // "ucs2_turkish_ci",
+        {138, "ucs2", true}, // "ucs2_czech_ci",
+        {139, "ucs2", true}, // "ucs2_danish_ci",
+        {140, "ucs2", true}, // "ucs2_lithuanian_ci",
+        {141, "ucs2", true}, // "ucs2_slovak_ci",
+        {142, "ucs2", true}, // "ucs2_spanish2_ci",
+        {143, "ucs2", true}, // "ucs2_roman_ci",
+        {144, "ucs2", true}, // "ucs2_persian_ci",
+        {145, "ucs2", true}, // "ucs2_esperanto_ci",
+        {146, "ucs2", true}, // "ucs2_hungarian_ci",
+        {147, "ucs2", true}, // "ucs2_sinhala_ci",
+        {148, "ucs2", true}, // "ucs2_german2_ci",
+        {149, "ucs2", true}, // "ucs2_croatian_ci",
+        {150, "ucs2", true}, // "ucs2_unicode_520_ci",
+        {151, "ucs2", true}, // "ucs2_vietnamese_ci",
+        {159, "ucs2", true}, // "ucs2_general_mysql500_ci",
+        {160, "utf32", true}, // "utf32_unicode_ci",
+        {161, "utf32", true}, // "utf32_icelandic_ci",
+        {162, "utf32", true}, // "utf32_latvian_ci",
+        {163, "utf32", true}, // "utf32_romanian_ci",
+        {164, "utf32", true}, // "utf32_slovenian_ci",
+        {165, "utf32", true}, // "utf32_polish_ci",
+        {166, "utf32", true}, // "utf32_estonian_ci",
+        {167, "utf32", true}, // "utf32_spanish_ci",
+        {168, "utf32", true}, // "utf32_swedish_ci",
+        {169, "utf32", true}, // "utf32_turkish_ci",
+        {170, "utf32", true}, // "utf32_czech_ci",
+        {171, "utf32", true}, // "utf32_danish_ci",
+        {172, "utf32", true}, // "utf32_lithuanian_ci",
+        {173, "utf32", true}, // "utf32_slovak_ci",
+        {174, "utf32", true}, // "utf32_spanish2_ci",
+        {175, "utf32", true}, // "utf32_roman_ci",
+        {176, "utf32", true}, // "utf32_persian_ci",
+        {177, "utf32", true}, // "utf32_esperanto_ci",
+        {178, "utf32", true}, // "utf32_hungarian_ci",
+        {179, "utf32", true}, // "utf32_sinhala_ci",
+        {180, "utf32", true}, // "utf32_german2_ci",
+        {181, "utf32", true}, // "utf32_croatian_ci",
+        {182, "utf32", true}, // "utf32_unicode_520_ci",
+        {183, "utf32", true}, // "utf32_vietnamese_ci",
+        {192, "utf8", false}, // "utf8_unicode_ci",
+        {193, "utf8", false}, // "utf8_icelandic_ci",
+        {194, "utf8", false}, // "utf8_latvian_ci",
+        {195, "utf8", false}, // "utf8_romanian_ci",
+        {196, "utf8", false}, // "utf8_slovenian_ci",
+        {197, "utf8", false}, // "utf8_polish_ci",
+        {198, "utf8", false}, // "utf8_estonian_ci",
+        {199, "utf8", false}, // "utf8_spanish_ci",
+        {200, "utf8", false}, // "utf8_swedish_ci",
+        {201, "utf8", false}, // "utf8_turkish_ci",
+        {202, "utf8", false}, // "utf8_czech_ci",
+        {203, "utf8", false}, // "utf8_danish_ci",
+        {204, "utf8", false}, // "utf8_lithuanian_ci",
+        {205, "utf8", false}, // "utf8_slovak_ci",
+        {206, "utf8", false}, // "utf8_spanish2_ci",
+        {207, "utf8", false}, // "utf8_roman_ci",
+        {208, "utf8", false}, // "utf8_persian_ci",
+        {209, "utf8", false}, // "utf8_esperanto_ci",
+        {210, "utf8", false}, // "utf8_hungarian_ci",
+        {211, "utf8", false}, // "utf8_sinhala_ci",
+        {212, "utf8", false}, // "utf8_german2_ci",
+        {213, "utf8", false}, // "utf8_croatian_ci",
+        {214, "utf8", false}, // "utf8_unicode_520_ci",
+        {215, "utf8", false}, // "utf8_vietnamese_ci",
+        {223, "utf8", false}, // "utf8_general_mysql500_ci",
+        {224, "utf8mb4", false}, // "utf8mb4_unicode_ci",
+        {225, "utf8mb4", false}, // "utf8mb4_icelandic_ci",
+        {226, "utf8mb4", false}, // "utf8mb4_latvian_ci",
+        {227, "utf8mb4", false}, // "utf8mb4_romanian_ci",
+        {228, "utf8mb4", false}, // "utf8mb4_slovenian_ci",
+        {229, "utf8mb4", false}, // "utf8mb4_polish_ci",
+        {230, "utf8mb4", false}, // "utf8mb4_estonian_ci",
+        {231, "utf8mb4", false}, // "utf8mb4_spanish_ci",
+        {232, "utf8mb4", false}, // "utf8mb4_swedish_ci",
+        {233, "utf8mb4", false}, // "utf8mb4_turkish_ci",
+        {234, "utf8mb4", false}, // "utf8mb4_czech_ci",
+        {235, "utf8mb4", false}, // "utf8mb4_danish_ci",
+        {236, "utf8mb4", false}, // "utf8mb4_lithuanian_ci",
+        {237, "utf8mb4", false}, // "utf8mb4_slovak_ci",
+        {238, "utf8mb4", false}, // "utf8mb4_spanish2_ci",
+        {239, "utf8mb4", false}, // "utf8mb4_roman_ci",
+        {240, "utf8mb4", false}, // "utf8mb4_persian_ci",
+        {241, "utf8mb4", false}, // "utf8mb4_esperanto_ci",
+        {242, "utf8mb4", false}, // "utf8mb4_hungarian_ci",
+        {243, "utf8mb4", false}, // "utf8mb4_sinhala_ci",
+        {244, "utf8mb4", false}, // "utf8mb4_german2_ci",
+        {245, "utf8mb4", false}, // "utf8mb4_croatian_ci",
+        {246, "utf8mb4", false}, // "utf8mb4_unicode_520_ci",
+        {247, "utf8mb4", false}, // "utf8mb4_vietnamese_ci",
+        {248, "gb18030", true}, // "gb18030_chinese_ci",
+        {249, "gb18030", true}, // "gb18030_bin",
+        {250, "gb18030", true}, // "gb18030_unicode_520_ci",
+        {255, "utf8mb4", false}, // "utf8mb4_0900_ai_ci",
+        {256, "utf8mb4", false}, // "utf8mb4_de_pb_0900_ai_ci",
+        {257, "utf8mb4", false}, // "utf8mb4_is_0900_ai_ci",
+        {258, "utf8mb4", false}, // "utf8mb4_lv_0900_ai_ci",
+        {259, "utf8mb4", false}, // "utf8mb4_ro_0900_ai_ci",
+        {260, "utf8mb4", false}, // "utf8mb4_sl_0900_ai_ci",
+        {261, "utf8mb4", false}, // "utf8mb4_pl_0900_ai_ci",
+        {262, "utf8mb4", false}, // "utf8mb4_et_0900_ai_ci",
+        {263, "utf8mb4", false}, // "utf8mb4_es_0900_ai_ci",
+        {264, "utf8mb4", false}, // "utf8mb4_is_0900_ai_ci",
+        {265, "utf8mb4", false}, // "utf8mb4_tr_0900_ai_ci",
+        {266, "utf8mb4", false}, // "utf8mb4_cs_0900_ai_ci",
+        {267, "utf8mb4", false}, // "utf8mb4_da_0900_ai_ci",
+        {268, "utf8mb4", false}, // "utf8mb4_lt_0900_ai_ci",
+        {269, "utf8mb4", false}, // "utf8mb4_sk_0900_ai_ci",
+        {270, "utf8mb4", false}, // "utf8mb4_es_trad_0900_ai_ci",
+        {271, "utf8mb4", false}, // "utf8mb4_la_0900_ai_ci",
+        {272, "utf8mb4", false}, // "utf8mb4_fa_0900_ai_ci",
+        {273, "utf8mb4", false}, // "utf8mb4_eo_0900_ai_ci",
+        {274, "utf8mb4", false}, // "utf8mb4_hu_0900_ai_ci",
+        {275, "utf8mb4", false}, // "utf8mb4_hr_0900_ai_ci",
+        {276, "utf8mb4", false}, // "utf8mb4_si_0900_ai_ci",
+        {277, "utf8mb4", false}, // "utf8mb4_vi_0900_ai_ci",
+        {278, "utf8mb4", false}, // "utf8mb4_0900_as_cs",
+        {279, "utf8mb4", false}, // "utf8mb4_de_pb_0900_as_cs",
+        {280, "utf8mb4", false}, // "utf8mb4_is_0900_as_cs",
+        {281, "utf8mb4", false}, // "utf8mb4_lv_0900_as_cs",
+        {282, "utf8mb4", false}, // "utf8mb4_ro_0900_as_cs",
+        {283, "utf8mb4", false}, // "utf8mb4_sl_0900_as_cs",
+        {284, "utf8mb4", false}, // "utf8mb4_pl_0900_as_cs",
+        {285, "utf8mb4", false}, // "utf8mb4_et_0900_as_cs",
+        {286, "utf8mb4", false}, // "utf8mb4_es_0900_as_cs",
+        {287, "utf8mb4", false}, // "utf8mb4_sv_0900_as_cs",
+        {288, "utf8mb4", false}, // "utf8mb4_tr_0900_as_cs",
+        {289, "utf8mb4", false}, // "utf8mb4_cs_0900_as_cs",
+        {290, "utf8mb4", false}, // "utf8mb4_da_0900_as_cs"
+        {291, "utf8mb4", false}, // "utf8mb4_lt_0900_as_cs"
+        {292, "utf8mb4", false}, // "utf8mb4_sk_0900_as_cs"
+        {293, "utf8mb4", false}, // "utf8mb4_es_trad_0900_as_cs"
+        {294, "utf8mb4", false}, // "utf8mb4_la_0900_as_cs"
+        {295, "utf8mb4", false}, // "utf8mb4_fa_0900_as_cs"
+        {296, "utf8mb4", false}, // "utf8mb4_eo_0900_as_cs"
+        {297, "utf8mb4", false}, // "utf8mb4_hu_0900_as_cs"
+        {298, "utf8mb4", false}, // "utf8mb4_hr_0900_as_cs"
+        {299, "utf8mb4", false}, // "utf8mb4_si_0900_as_cs"
+        {300, "utf8mb4", false}, // "utf8mb4_vi_0900_as_cs"
+        {303, "utf8mb4", false}, // "utf8mb4_ja_0900_as_cs_ks"
+        {304, "utf8mb4", false}, // "utf8mb4_la_0900_as_cs"
+        {305, "utf8mb4", false}, // "utf8mb4_0900_as_ci"
+        {306, "utf8mb4", false}, // "utf8mb4_ru_0900_ai_ci"
+        {307, "utf8mb4", false}, // "utf8mb4_ru_0900_as_cs"
+        {308, "utf8mb4", false}, // "utf8mb4_zh_0900_as_cs"
+        {309, "utf8mb4", false} // "utf8mb4_0900_bin"
+    };
+
+    MySQLCharset charset;
+
+    for (auto & item : result)
+    {
+        EXPECT_TRUE(charset.needConvert(item.id) == item.need_convert);
+        if (charset.needConvert(item.id))
+        {
+            EXPECT_TRUE(charset.getCharsetFromId(item.id) == item.name);
+        }
+    }
+}
+
+}

src/Databases/DatabaseAtomic.cpp

@@ -441,11 +441,10 @@ void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, Loadin
     }
 }
 
-void DatabaseAtomic::loadStoredObjects(
-    ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
+void DatabaseAtomic::loadStoredObjects(ContextMutablePtr local_context, LoadingStrictnessLevel mode)
 {
     beforeLoadingMetadata(local_context, mode);
-    DatabaseOrdinary::loadStoredObjects(local_context, mode, skip_startup_tables);
+    DatabaseOrdinary::loadStoredObjects(local_context, mode);
 }
 
 void DatabaseAtomic::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)

src/Databases/DatabaseAtomic.h

@@ -48,7 +48,7 @@ public:
 
     DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;
 
-    void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
+    void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
 
     void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
 

src/Databases/DatabaseLazy.cpp

@@ -37,8 +37,7 @@ DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_,
 }
 
 
-void DatabaseLazy::loadStoredObjects(
-    ContextMutablePtr local_context, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
+void DatabaseLazy::loadStoredObjects(ContextMutablePtr local_context, LoadingStrictnessLevel /*mode*/)
 {
     iterateMetadataFiles(local_context, [this, &local_context](const String & file_name)
     {

src/Databases/DatabaseLazy.h

@@ -26,7 +26,7 @@ public:
 
     bool canContainDistributedTables() const override { return false; }
 
-    void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
+    void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel /*mode*/) override;
 
     void createTable(
         ContextPtr context,

src/Databases/DatabaseOrdinary.cpp

@@ -89,8 +89,7 @@ DatabaseOrdinary::DatabaseOrdinary(
 {
 }
 
-void DatabaseOrdinary::loadStoredObjects(
-    ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
+void DatabaseOrdinary::loadStoredObjects(ContextMutablePtr local_context, LoadingStrictnessLevel mode)
 {
     /** Tables load faster if they are loaded in sorted (by name) order.
       * Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order,
@@ -159,12 +158,6 @@ void DatabaseOrdinary::loadStoredObjects(
     }
 
     pool.wait();
-
-    if (!skip_startup_tables)
-    {
-        /// After all tables was basically initialized, startup them.
-        startupTables(pool, mode);
-    }
 }
 
 void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTablesMetadata & metadata, bool is_startup)

src/Databases/DatabaseOrdinary.h

@@ -21,7 +21,7 @@ public:
 
     String getEngineName() const override { return "Ordinary"; }
 
-    void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
+    void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
 
     bool supportsLoadingInTopologicalOrder() const override { return true; }
 

src/Databases/DatabaseReplicated.cpp

@@ -495,11 +495,10 @@ void DatabaseReplicated::beforeLoadingMetadata(ContextMutablePtr /*context*/, Lo
     tryConnectToZooKeeperAndInitDatabase(mode);
 }
 
-void DatabaseReplicated::loadStoredObjects(
-    ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
+void DatabaseReplicated::loadStoredObjects(ContextMutablePtr local_context, LoadingStrictnessLevel mode)
 {
     beforeLoadingMetadata(local_context, mode);
-    DatabaseAtomic::loadStoredObjects(local_context, mode, skip_startup_tables);
+    DatabaseAtomic::loadStoredObjects(local_context, mode);
 }
 
 UInt64 DatabaseReplicated::getMetadataHash(const String & table_name) const

src/Databases/DatabaseReplicated.h

@@ -67,7 +67,7 @@ public:
 
     void drop(ContextPtr /*context*/) override;
 
-    void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
+    void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
 
     void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
 

src/Databases/IDatabase.h

@@ -134,8 +134,7 @@ public:
     /// You can call only once, right after the object is created.
     virtual void loadStoredObjects( /// NOLINT
         ContextMutablePtr /*context*/,
-        LoadingStrictnessLevel /*mode*/,
-        bool /* skip_startup_tables */)
+        LoadingStrictnessLevel /*mode*/)
     {
     }
 

src/Databases/MySQL/DatabaseMySQL.cpp

@@ -402,7 +402,7 @@ String DatabaseMySQL::getMetadataPath() const
     return metadata_path;
 }
 
-void DatabaseMySQL::loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
+void DatabaseMySQL::loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/)
 {
 
     std::lock_guard lock{mutex};

src/Databases/MySQL/DatabaseMySQL.h

@@ -76,7 +76,7 @@ public:
 
     void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override;
 
-    void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
+    void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/) override;
 
     StoragePtr detachTable(ContextPtr context, const String & table_name) override;
 

src/Databases/PostgreSQL/DatabasePostgreSQL.cpp

@@ -296,7 +296,7 @@ void DatabasePostgreSQL::drop(ContextPtr /*context*/)
 }
 
 
-void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
+void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, LoadingStrictnessLevel /*mode*/)
 {
     {
         std::lock_guard lock{mutex};

src/Databases/PostgreSQL/DatabasePostgreSQL.h

@@ -44,7 +44,7 @@ public:
 
     bool empty() const override;
 
-    void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
+    void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/) override;
 
     DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;
 

src/Databases/TablesLoader.cpp

@@ -49,7 +49,7 @@ void TablesLoader::loadTables()
         if (need_resolve_dependencies && database.second->supportsLoadingInTopologicalOrder())
             databases_to_load.push_back(database.first);
         else
-            database.second->loadStoredObjects(global_context, strictness_mode, /* skip_startup_tables */ true);
+            database.second->loadStoredObjects(global_context, strictness_mode);
     }
 
     if (databases_to_load.empty())

src/Disks/ObjectStorages/Web/MetadataStorageFromStaticFilesWebServer.cpp

@@ -33,46 +33,18 @@ const std::string & MetadataStorageFromStaticFilesWebServer::getPath() const
 
 bool MetadataStorageFromStaticFilesWebServer::exists(const std::string & path) const
 {
-    fs::path fs_path(path);
-    if (fs_path.has_extension())
-        fs_path = fs_path.parent_path();
-
-    initializeIfNeeded(fs_path);
-
-    if (object_storage.files.empty())
-        return false;
-
-    if (object_storage.files.contains(path))
-        return true;
-
-    /// `object_storage.files` contains files + directories only inside `metadata_path / uuid_3_digit / uuid /`
-    /// (specific table files only), but we need to be able to also tell if `exists(<metadata_path>)`, for example.
-    auto it = std::lower_bound(
-        object_storage.files.begin(),
-        object_storage.files.end(),
-        path,
-        [](const auto & file, const std::string & path_) { return file.first < path_; }
-    );
-    if (it == object_storage.files.end())
-        return false;
-
-    if (startsWith(it->first, path)
-        || (it != object_storage.files.begin() && startsWith(std::prev(it)->first, path)))
-        return true;
-
-    return false;
+    return object_storage.exists(path);
 }
 
 void MetadataStorageFromStaticFilesWebServer::assertExists(const std::string & path) const
 {
-    initializeIfNeeded(path);
-
     if (!exists(path))
 #ifdef NDEBUG
         throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "There is no path {}", path);
 #else
     {
         std::string all_files;
+        std::shared_lock shared_lock(object_storage.metadata_mutex);
         for (const auto & [file, _] : object_storage.files)
         {
             if (!all_files.empty())
@@ -87,33 +59,40 @@ void MetadataStorageFromStaticFilesWebServer::assertExists(const std::string & p
 bool MetadataStorageFromStaticFilesWebServer::isFile(const std::string & path) const
 {
     assertExists(path);
+    std::shared_lock shared_lock(object_storage.metadata_mutex);
     return object_storage.files.at(path).type == WebObjectStorage::FileType::File;
 }
 
 bool MetadataStorageFromStaticFilesWebServer::isDirectory(const std::string & path) const
 {
     assertExists(path);
+    std::shared_lock shared_lock(object_storage.metadata_mutex);
     return object_storage.files.at(path).type == WebObjectStorage::FileType::Directory;
 }
 
 uint64_t MetadataStorageFromStaticFilesWebServer::getFileSize(const String & path) const
 {
     assertExists(path);
+    std::shared_lock shared_lock(object_storage.metadata_mutex);
     return object_storage.files.at(path).size;
 }
 
 StoredObjects MetadataStorageFromStaticFilesWebServer::getStorageObjects(const std::string & path) const
 {
     assertExists(path);
+
     auto fs_path = fs::path(object_storage.url) / path;
     std::string remote_path = fs_path.parent_path() / (escapeForFileName(fs_path.stem()) + fs_path.extension().string());
     remote_path = remote_path.substr(object_storage.url.size());
+
+    std::shared_lock shared_lock(object_storage.metadata_mutex);
     return {StoredObject(remote_path, object_storage.files.at(path).size, path)};
 }
 
 std::vector<std::string> MetadataStorageFromStaticFilesWebServer::listDirectory(const std::string & path) const
 {
     std::vector<std::string> result;
+    std::shared_lock shared_lock(object_storage.metadata_mutex);
     for (const auto & [file_path, _] : object_storage.files)
     {
         if (file_path.starts_with(path))
@@ -122,22 +101,14 @@ std::vector<std::string> MetadataStorageFromStaticFilesWebServer::listDirectory(
     return result;
 }
 
-void MetadataStorageFromStaticFilesWebServer::initializeIfNeeded(const std::string & path) const
-{
-    if (object_storage.files.find(path) == object_storage.files.end())
-    {
-        object_storage.initialize(fs::path(object_storage.url) / path);
-    }
-}
-
 DirectoryIteratorPtr MetadataStorageFromStaticFilesWebServer::iterateDirectory(const std::string & path) const
 {
     std::vector<fs::path> dir_file_paths;
 
-    initializeIfNeeded(path);
     if (!exists(path))
         return std::make_unique<StaticDirectoryIterator>(std::move(dir_file_paths));
 
+    std::shared_lock shared_lock(object_storage.metadata_mutex);
     for (const auto & [file_path, _] : object_storage.files)
     {
         if (fs::path(parentPath(file_path)) / "" == fs::path(path) / "")

src/Disks/ObjectStorages/Web/MetadataStorageFromStaticFilesWebServer.h

@@ -13,13 +13,14 @@ class MetadataStorageFromStaticFilesWebServer final : public IMetadataStorage
 {
 private:
     friend class MetadataStorageFromStaticFilesWebServerTransaction;
+    using FileType = WebObjectStorage::FileType;
 
     const WebObjectStorage & object_storage;
     std::string root_path;
 
     void assertExists(const std::string & path) const;
 
-    void initializeIfNeeded(const std::string & path) const;
+    void initializeImpl(const String & uri_path, const std::unique_lock<std::shared_mutex> &) const;
 
 public:
     explicit MetadataStorageFromStaticFilesWebServer(const WebObjectStorage & object_storage_);

src/Disks/ObjectStorages/Web/WebObjectStorage.cpp

@@ -28,10 +28,9 @@ namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
     extern const int NOT_IMPLEMENTED;
-    extern const int NETWORK_ERROR;
 }
 
-void WebObjectStorage::initialize(const String & uri_path) const
+void WebObjectStorage::initialize(const String & uri_path, const std::unique_lock<std::shared_mutex> & lock) const
 {
     std::vector<String> directories_to_load;
     LOG_TRACE(log, "Loading metadata for directory: {}", uri_path);
@@ -81,8 +80,9 @@ void WebObjectStorage::initialize(const String & uri_path) const
             }
 
             file_path = file_path.substr(url.size());
-            files.emplace(std::make_pair(file_path, file_data));
             LOG_TRACE(&Poco::Logger::get("DiskWeb"), "Adding file: {}, size: {}", file_path, file_data.size);
+
+            files.emplace(std::make_pair(file_path, file_data));
         }
 
         files.emplace(std::make_pair(dir_name, FileData({ .type = FileType::Directory })));
@@ -103,7 +103,7 @@ void WebObjectStorage::initialize(const String & uri_path) const
     }
 
     for (const auto & directory_path : directories_to_load)
-        initialize(directory_path);
+        initialize(directory_path, lock);
 }
 
 
@@ -118,31 +118,51 @@ WebObjectStorage::WebObjectStorage(
 
 bool WebObjectStorage::exists(const StoredObject & object) const
 {
-    const auto & path = object.remote_path;
+    return exists(object.remote_path);
+}
 
+bool WebObjectStorage::exists(const std::string & path) const
+{
     LOG_TRACE(&Poco::Logger::get("DiskWeb"), "Checking existence of path: {}", path);
 
-    if (files.find(path) != files.end())
+    std::shared_lock shared_lock(metadata_mutex);
+
+    if (files.find(path) == files.end())
+    {
+        shared_lock.unlock();
+        std::unique_lock unique_lock(metadata_mutex);
+        if (files.find(path) == files.end())
+        {
+            fs::path index_file_dir = fs::path(url) / path;
+            if (index_file_dir.has_extension())
+                index_file_dir = index_file_dir.parent_path();
+
+            initialize(index_file_dir, unique_lock);
+        }
+        /// Files are never deleted from `files` as disk is read only, so no worry that we unlock now.
+        unique_lock.unlock();
+        shared_lock.lock();
+    }
+
+    if (files.empty())
+        return false;
+
+    if (files.contains(path))
         return true;
 
-    if (path.ends_with(MergeTreeData::FORMAT_VERSION_FILE_NAME) && files.find(fs::path(path).parent_path() / "") == files.end())
-    {
-        try
-        {
-            initialize(fs::path(url) / fs::path(path).parent_path());
-            return files.find(path) != files.end();
-        }
-        catch (...)
-        {
-            const auto message = getCurrentExceptionMessage(false);
-            bool can_throw = CurrentThread::isInitialized() && CurrentThread::get().getQueryContext();
-            if (can_throw)
-                throw Exception(ErrorCodes::NETWORK_ERROR, "Cannot load disk metadata. Error: {}", message);
+    /// `object_storage.files` contains files + directories only inside `metadata_path / uuid_3_digit / uuid /`
+    /// (specific table files only), but we need to be able to also tell if `exists(<metadata_path>)`, for example.
+    auto it = std::lower_bound(
+        files.begin(), files.end(), path,
+        [](const auto & file, const std::string & path_) { return file.first < path_; }
+    );
 
-            LOG_TRACE(&Poco::Logger::get("DiskWeb"), "Cannot load disk metadata. Error: {}", message);
-            return false;
-        }
-    }
+    if (it == files.end())
+        return false;
+
+    if (startsWith(it->first, path)
+        || (it != files.begin() && startsWith(std::prev(it)->first, path)))
+        return true;
 
     return false;
 }

src/Disks/ObjectStorages/Web/WebObjectStorage.h

@@ -3,6 +3,7 @@
 #include "config.h"
 
 #include <Disks/ObjectStorages/IObjectStorage.h>
+#include <shared_mutex>
 
 namespace Poco
 {
@@ -93,9 +94,8 @@ public:
     bool isReadOnly() const override { return true; }
 
 protected:
-    void initialize(const String & uri_path) const;
-
     [[noreturn]] static void throwNotAllowed();
+    bool exists(const std::string & path) const;
 
     enum class FileType
     {
@@ -111,12 +111,13 @@ protected:
 
     using Files = std::map<String, FileData>; /// file path -> file data
     mutable Files files;
-
-    String url;
+    mutable std::shared_mutex metadata_mutex;
 
 private:
-    Poco::Logger * log;
+    void initialize(const String & path, const std::unique_lock<std::shared_mutex> &) const;
 
+    const String url;
+    Poco::Logger * log;
     size_t min_bytes_for_seek;
 };
 

src/Formats/ReadSchemaUtils.h

@@ -9,14 +9,14 @@ namespace DB
 
 using ReadBufferIterator = std::function<std::unique_ptr<ReadBuffer>(ColumnsDescription &)>;
 
-/// Try to determine the schema of the data in specifying format.
+/// Try to determine the schema of the data in the specified format.
 /// For formats that have an external schema reader, it will
 /// use it and won't create a read buffer.
 /// For formats that have a schema reader from the data,
 /// read buffer will be created by the provided iterator and
 /// the schema will be extracted from the data. If schema reader
 /// couldn't determine the schema we will try the next read buffer
-/// from provided iterator if it makes sense. If format doesn't
+/// from the provided iterator if it makes sense. If the format doesn't
 /// have any schema reader or we couldn't determine the schema,
 /// an exception will be thrown.
 ColumnsDescription readSchemaFromFormat(

src/Functions/FunctionToDecimalString.cpp

@@ -1,22 +0,0 @@
-#include <Functions/FunctionFactory.h>
-#include <Functions/FunctionToDecimalString.h>
-#include <Functions/IFunction.h>
-
-namespace DB
-{
-
-REGISTER_FUNCTION(ToDecimalString)
-{
-    factory.registerFunction<FunctionToDecimalString>(
-        FunctionDocumentation{
-            .description=R"(
-Returns string representation of a number. First argument is the number of any numeric type,
-second argument is the desired number of digits in fractional part. Returns String.
-
-        )",
-            .examples{{"toDecimalString", "SELECT toDecimalString(2.1456,2)", ""}},
-            .categories{"String"}
-        }, FunctionFactory::CaseInsensitive);
-}
-
-}

src/Functions/FunctionToDecimalString.h

@@ -1,312 +0,0 @@
-#pragma once
-
-#include <Core/Types.h>
-#include <Core/DecimalFunctions.h>
-#include <Functions/IFunction.h>
-#include <Functions/FunctionHelpers.h>
-#include <Columns/ColumnsNumber.h>
-#include <Columns/ColumnString.h>
-#include <Columns/ColumnVector.h>
-#include <Columns/ColumnDecimal.h>
-#include <DataTypes/DataTypeString.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <IO/WriteBufferFromVector.h>
-#include <IO/WriteHelpers.h>
-#include <Interpreters/Context_fwd.h>
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
-    extern const int ILLEGAL_COLUMN;
-    extern const int CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER;
-}
-
-class FunctionToDecimalString : public IFunction
-{
-public:
-    static constexpr auto name = "toDecimalString";
-    static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionToDecimalString>(); }
-
-    String getName() const override { return name; }
-
-    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
-
-    size_t getNumberOfArguments() const override { return 2; }
-
-    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
-    {
-        if (!isNumber(*arguments[0]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                            "Illegal first argument for formatDecimal function: got {}, expected numeric type",
-                            arguments[0]->getName());
-
-        if (!isUInt8(*arguments[1]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                            "Illegal second argument for formatDecimal function: got {}, expected UInt8",
-                            arguments[1]->getName());
-
-        return std::make_shared<DataTypeString>();
-    }
-
-    bool useDefaultImplementationForConstants() const override { return true; }
-
-private:
-    /// For operations with Integer/Float
-    template <typename FromVectorType>
-    void vectorConstant(const FromVectorType & vec_from, UInt8 precision,
-                        ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets) const
-    {
-        size_t input_rows_count = vec_from.size();
-        result_offsets.resize(input_rows_count);
-
-        /// Buffer is used here and in functions below because resulting size cannot be precisely anticipated,
-        /// and buffer resizes on-the-go. Also, .count() provided by buffer is convenient in this case.
-        WriteBufferFromVector<ColumnString::Chars> buf_to(vec_to);
-
-        for (size_t i = 0; i < input_rows_count; ++i)
-        {
-            format(vec_from[i], buf_to, precision);
-            result_offsets[i] = buf_to.count();
-        }
-
-        buf_to.finalize();
-    }
-
-    template <typename FirstArgVectorType>
-    void vectorVector(const FirstArgVectorType & vec_from, const ColumnVector<UInt8>::Container & vec_precision,
-                      ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets) const
-    {
-        size_t input_rows_count = vec_from.size();
-        result_offsets.resize(input_rows_count);
-
-        WriteBufferFromVector<ColumnString::Chars> buf_to(vec_to);
-
-        constexpr size_t max_digits = std::numeric_limits<UInt256>::digits10;
-
-        for (size_t i = 0; i < input_rows_count; ++i)
-        {
-            if (vec_precision[i] > max_digits)
-                throw DB::Exception(DB::ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER,
-                                    "Too many fractional digits requested, shall not be more than {}", max_digits);
-            format(vec_from[i], buf_to, vec_precision[i]);
-            result_offsets[i] = buf_to.count();
-        }
-
-        buf_to.finalize();
-    }
-
-    template <typename FirstArgType>
-    void constantVector(const FirstArgType & value_from, const ColumnVector<UInt8>::Container & vec_precision,
-                        ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets) const
-    {
-        size_t input_rows_count = vec_precision.size();
-        result_offsets.resize(input_rows_count);
-
-        WriteBufferFromVector<ColumnString::Chars> buf_to(vec_to);
-
-        constexpr size_t max_digits = std::numeric_limits<UInt256>::digits10;
-
-        for (size_t i = 0; i < input_rows_count; ++i)
-        {
-            if (vec_precision[i] > max_digits)
-                throw DB::Exception(DB::ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER,
-                                    "Too many fractional digits requested, shall not be more than {}", max_digits);
-            format(value_from, buf_to, vec_precision[i]);
-            result_offsets[i] = buf_to.count();
-        }
-
-        buf_to.finalize();
-    }
-
-    /// For operations with Decimal
-    template <typename FirstArgVectorType>
-    void vectorConstant(const FirstArgVectorType & vec_from, UInt8 precision,
-                        ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets, UInt8 from_scale) const
-    {
-        /// There are no more than 77 meaning digits (as it is the max length of UInt256). So we can limit it with 77.
-        constexpr size_t max_digits = std::numeric_limits<UInt256>::digits10;
-        if (precision > max_digits)
-            throw DB::Exception(DB::ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER,
-                                "Too many fractional digits requested for Decimal, must not be more than {}", max_digits);
-
-        WriteBufferFromVector<ColumnString::Chars> buf_to(vec_to);
-        size_t input_rows_count = vec_from.size();
-        result_offsets.resize(input_rows_count);
-
-        for (size_t i = 0; i < input_rows_count; ++i)
-        {
-            writeText(vec_from[i], from_scale, buf_to, true, true, precision);
-            writeChar(0, buf_to);
-            result_offsets[i] = buf_to.count();
-        }
-        buf_to.finalize();
-    }
-
-    template <typename FirstArgVectorType>
-    void vectorVector(const FirstArgVectorType & vec_from, const ColumnVector<UInt8>::Container & vec_precision,
-                      ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets, UInt8 from_scale) const
-    {
-        size_t input_rows_count = vec_from.size();
-        result_offsets.resize(input_rows_count);
-
-        WriteBufferFromVector<ColumnString::Chars> buf_to(vec_to);
-
-        constexpr size_t max_digits = std::numeric_limits<UInt256>::digits10;
-
-        for (size_t i = 0; i < input_rows_count; ++i)
-        {
-            if (vec_precision[i] > max_digits)
-                throw DB::Exception(DB::ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER,
-                                    "Too many fractional digits requested for Decimal, must not be more than {}", max_digits);
-            writeText(vec_from[i], from_scale, buf_to, true, true, vec_precision[i]);
-            writeChar(0, buf_to);
-            result_offsets[i] = buf_to.count();
-        }
-        buf_to.finalize();
-    }
-
-    template <typename FirstArgType>
-    void constantVector(const FirstArgType & value_from, const ColumnVector<UInt8>::Container & vec_precision,
-                        ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets, UInt8 from_scale) const
-    {
-        size_t input_rows_count = vec_precision.size();
-        result_offsets.resize(input_rows_count);
-
-        WriteBufferFromVector<ColumnString::Chars> buf_to(vec_to);
-
-        constexpr size_t max_digits = std::numeric_limits<UInt256>::digits10;
-
-        for (size_t i = 0; i < input_rows_count; ++i)
-        {
-            if (vec_precision[i] > max_digits)
-                throw DB::Exception(DB::ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER,
-                                    "Too many fractional digits requested for Decimal, must not be more than {}", max_digits);
-            writeText(value_from, from_scale, buf_to, true, true, vec_precision[i]);
-            writeChar(0, buf_to);
-            result_offsets[i] = buf_to.count();
-        }
-        buf_to.finalize();
-    }
-
-    template <is_floating_point T>
-    static void format(T value, DB::WriteBuffer & out, UInt8 precision)
-    {
-        /// Maximum of 60 is hard-coded in 'double-conversion/double-conversion.h' for floating point values,
-        /// Catch this here to give user a more reasonable error.
-        if (precision > 60)
-            throw DB::Exception(DB::ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER,
-                                "Too high precision requested for Float, must not be more than 60, got {}", Int8(precision));
-
-        DB::DoubleConverter<false>::BufferType buffer;
-        double_conversion::StringBuilder builder{buffer, sizeof(buffer)};
-
-        const auto result = DB::DoubleConverter<false>::instance().ToFixed(value, precision, &builder);
-
-        if (!result)
-            throw DB::Exception(DB::ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER, "Error processing number: {}", value);
-
-        out.write(buffer, builder.position());
-        writeChar(0, out);
-    }
-
-    template <is_integer T>
-    static void format(T value, DB::WriteBuffer & out, UInt8 precision)
-    {
-        /// Fractional part for Integer is just trailing zeros. Let's limit it with 77 (like with Decimals).
-        constexpr size_t max_digits = std::numeric_limits<UInt256>::digits10;
-        if (precision > max_digits)
-            throw DB::Exception(DB::ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER,
-                                "Too many fractional digits requested, shall not be more than {}", max_digits);
-        writeText(value, out);
-        if (precision > 0) [[likely]]
-        {
-            writeChar('.', out);
-            for (int i = 0; i < precision; ++i)
-                writeChar('0', out);
-            writeChar(0, out);
-        }
-    }
-
-public:
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
-    {
-        switch (arguments[0].type->getTypeId())
-        {
-            case TypeIndex::UInt8:      return executeType<UInt8>(arguments);
-            case TypeIndex::UInt16:     return executeType<UInt16>(arguments);
-            case TypeIndex::UInt32:     return executeType<UInt32>(arguments);
-            case TypeIndex::UInt64:     return executeType<UInt64>(arguments);
-            case TypeIndex::UInt128:    return executeType<UInt128>(arguments);
-            case TypeIndex::UInt256:    return executeType<UInt256>(arguments);
-            case TypeIndex::Int8:       return executeType<Int8>(arguments);
-            case TypeIndex::Int16:      return executeType<Int16>(arguments);
-            case TypeIndex::Int32:      return executeType<Int32>(arguments);
-            case TypeIndex::Int64:      return executeType<Int64>(arguments);
-            case TypeIndex::Int128:     return executeType<Int128>(arguments);
-            case TypeIndex::Int256:     return executeType<Int256>(arguments);
-            case TypeIndex::Float32:    return executeType<Float32>(arguments);
-            case TypeIndex::Float64:    return executeType<Float64>(arguments);
-            case TypeIndex::Decimal32:  return executeType<Decimal32>(arguments);
-            case TypeIndex::Decimal64:  return executeType<Decimal64>(arguments);
-            case TypeIndex::Decimal128: return executeType<Decimal128>(arguments);
-            case TypeIndex::Decimal256: return executeType<Decimal256>(arguments);
-            default:
-                throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
-                                arguments[0].column->getName(), getName());
-        }
-    }
-
-private:
-    template <typename T>
-    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments) const
-    {
-        const auto * from_col_const = typeid_cast<const ColumnConst *>(arguments[0].column.get());
-        const auto * precision_col = checkAndGetColumn<ColumnVector<UInt8>>(arguments[1].column.get());
-        const auto * precision_col_const = typeid_cast<const ColumnConst *>(arguments[1].column.get());
-
-        auto result_col = ColumnString::create();
-        auto * result_col_string = assert_cast<ColumnString *>(result_col.get());
-        ColumnString::Chars & result_chars = result_col_string->getChars();
-        ColumnString::Offsets & result_offsets = result_col_string->getOffsets();
-
-        if constexpr (is_decimal<T>)
-        {
-            const auto * from_col = checkAndGetColumn<ColumnDecimal<T>>(arguments[0].column.get());
-            UInt8 from_scale = from_col->getScale();
-
-            if (from_col)
-            {
-                if (precision_col_const)
-                    vectorConstant(from_col->getData(), precision_col_const->template getValue<UInt8>(), result_chars, result_offsets, from_scale);
-                else
-                    vectorVector(from_col->getData(), precision_col->getData(), result_chars, result_offsets, from_scale);
-            }
-            else if (from_col_const)
-                constantVector(from_col_const->template getValue<T>(), precision_col->getData(), result_chars, result_offsets, from_scale);
-            else
-                throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function formatDecimal", arguments[0].column->getName());
-        }
-        else
-        {
-            const auto * from_col = checkAndGetColumn<ColumnVector<T>>(arguments[0].column.get());
-            if (from_col)
-            {
-                if (precision_col_const)
-                    vectorConstant(from_col->getData(), precision_col_const->template getValue<UInt8>(), result_chars, result_offsets);
-                else
-                    vectorVector(from_col->getData(), precision_col->getData(), result_chars, result_offsets);
-            }
-            else if (from_col_const)
-                constantVector(from_col_const->template getValue<T>(), precision_col->getData(), result_chars, result_offsets);
-            else
-                throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function formatDecimal", arguments[0].column->getName());
-        }
-
-        return result_col;
-    }
-};
-
-}

src/Functions/HasSubsequenceImpl.h

@@ -0,0 +1,158 @@
+#pragma once
+
+#include <Columns/ColumnString.h>
+#include <Columns/ColumnConst.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Functions/GatherUtils/Sources.h>
+#include <Functions/GatherUtils/Sinks.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int ILLEGAL_COLUMN;
+}
+namespace
+{
+
+using namespace GatherUtils;
+
+template <typename Name, typename Impl>
+class HasSubsequenceImpl : public IFunction
+{
+public:
+    static constexpr auto name = Name::name;
+
+    static FunctionPtr create(ContextPtr) { return std::make_shared<HasSubsequenceImpl>(); }
+
+    String getName() const override { return name; }
+
+    bool isVariadic() const override { return false; }
+
+    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
+
+    size_t getNumberOfArguments() const override { return 2; }
+
+    bool useDefaultImplementationForConstants() const override { return false; }
+
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {};}
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        if (!isString(arguments[0]))
+            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                "Illegal type {} of argument of function {}",
+                arguments[0]->getName(), getName());
+
+        if (!isString(arguments[1]))
+            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                "Illegal type {} of argument of function {}",
+                arguments[1]->getName(), getName());
+
+        return std::make_shared<DataTypeNumber<UInt8>>();
+    }
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override
+    {
+        const ColumnPtr & column_haystack = arguments[0].column;
+        const ColumnPtr & column_needle = arguments[1].column;
+
+        const ColumnConst * haystack_const_string = checkAndGetColumnConst<ColumnString>(column_haystack.get());
+        const ColumnConst * needle_const_string = checkAndGetColumnConst<ColumnString>(column_needle.get());
+        const ColumnString * haystack_string = checkAndGetColumn<ColumnString>(&*column_haystack);
+        const ColumnString * needle_string = checkAndGetColumn<ColumnString>(&*column_needle);
+
+        auto col_res = ColumnVector<UInt8>::create();
+        typename ColumnVector<UInt8>::Container & vec_res = col_res->getData();
+        vec_res.resize(input_rows_count);
+
+        if (haystack_string && needle_string)
+            execute(StringSource{*haystack_string}, StringSource{*needle_string}, vec_res);
+        else if (haystack_string && needle_const_string)
+            execute(StringSource{*haystack_string}, ConstSource<StringSource>{*needle_const_string}, vec_res);
+        else if (haystack_const_string && needle_string)
+            execute(ConstSource<StringSource>{*haystack_const_string}, StringSource{*needle_string}, vec_res);
+        else if (haystack_const_string && needle_const_string)
+            execute(ConstSource<StringSource>{*haystack_const_string}, ConstSource<StringSource>{*needle_const_string}, vec_res);
+        else
+            throw Exception(
+                ErrorCodes::ILLEGAL_COLUMN,
+                "Illegal columns {} and {} of arguments of function {}",
+                arguments[0].column->getName(),
+                arguments[1].column->getName(),
+                getName());
+
+        return col_res;
+    }
+
+private:
+
+    template <typename SourceHaystack, typename SourceNeedle>
+    void execute(
+        SourceHaystack && haystacks,
+        SourceNeedle && needles,
+        PaddedPODArray<UInt8> & res_data) const
+    {
+        while (!haystacks.isEnd())
+        {
+            auto haystack_slice = haystacks.getWhole();
+            auto needle_slice = needles.getWhole();
+            size_t row_num = haystacks.rowNum();
+
+            if constexpr (!Impl::is_utf8)
+                res_data[row_num] = hasSubsequence(haystack_slice.data, haystack_slice.size, needle_slice.data, needle_slice.size);
+            else
+                res_data[row_num] = hasSubsequenceUTF8(haystack_slice.data, haystack_slice.size, needle_slice.data, needle_slice.size);
+
+            haystacks.next();
+            needles.next();
+        }
+    }
+
+    static UInt8 hasSubsequence(const UInt8 * haystack, size_t haystack_size, const UInt8 * needle, size_t needle_size)
+    {
+        size_t j = 0;
+        for (size_t i = 0; (i < haystack_size) && (j < needle_size); i++)
+            if (Impl::toLowerIfNeed(needle[j]) == Impl::toLowerIfNeed(haystack[i]))
+                ++j;
+        return j == needle_size;
+    }
+
+    static UInt8 hasSubsequenceUTF8(const UInt8 * haystack, size_t haystack_size, const UInt8 * needle, size_t needle_size)
+    {
+        const auto * haystack_pos = haystack;
+        const auto * needle_pos = needle;
+        const auto * haystack_end = haystack + haystack_size;
+        const auto * needle_end = needle + needle_size;
+
+        if (!needle_size)
+            return 1;
+
+        auto haystack_code_point = UTF8::convertUTF8ToCodePoint(haystack_pos, haystack_end - haystack_pos);
+        auto needle_code_point = UTF8::convertUTF8ToCodePoint(needle_pos, needle_end - needle_pos);
+        if (!haystack_code_point || !needle_code_point)
+            return 0;
+
+        while (haystack_code_point && needle_code_point)
+        {
+            if (Impl::toLowerIfNeed(*needle_code_point) == Impl::toLowerIfNeed(*haystack_code_point))
+            {
+                needle_pos += UTF8::seqLength(*needle_pos);
+                if (needle_pos >= needle_end)
+                    break;
+                needle_code_point = UTF8::convertUTF8ToCodePoint(needle_pos, needle_end - needle_pos);
+            }
+            haystack_pos += UTF8::seqLength(*haystack_pos);
+            if (haystack_pos >= haystack_end)
+                break;
+            haystack_code_point = UTF8::convertUTF8ToCodePoint(haystack_pos, haystack_end - haystack_pos);
+        }
+        return needle_pos == needle_end;
+    }
+};
+
+}
+
+}

src/Functions/hasSubsequence.cpp

@@ -0,0 +1,30 @@
+#include <Functions/FunctionFactory.h>
+#include <Functions/HasSubsequenceImpl.h>
+
+
+namespace DB
+{
+namespace
+{
+
+struct HasSubsequenceCaseSensitiveASCII
+{
+    static constexpr bool is_utf8 = false;
+
+    static int toLowerIfNeed(int c) { return c; }
+};
+
+struct NameHasSubsequence
+{
+    static constexpr auto name = "hasSubsequence";
+};
+
+using FunctionHasSubsequence = HasSubsequenceImpl<NameHasSubsequence, HasSubsequenceCaseSensitiveASCII>;
+}
+
+REGISTER_FUNCTION(hasSubsequence)
+{
+    factory.registerFunction<FunctionHasSubsequence>({}, FunctionFactory::CaseInsensitive);
+}
+
+}

src/Functions/hasSubsequenceCaseInsensitive.cpp

@@ -0,0 +1,29 @@
+#include <Functions/FunctionFactory.h>
+#include <Functions/HasSubsequenceImpl.h>
+
+namespace DB
+{
+namespace
+{
+
+struct HasSubsequenceCaseInsensitiveASCII
+{
+    static constexpr bool is_utf8 = false;
+
+    static int toLowerIfNeed(int c) { return std::tolower(c); }
+};
+
+struct NameHasSubsequenceCaseInsensitive
+{
+    static constexpr auto name = "hasSubsequenceCaseInsensitive";
+};
+
+using FunctionHasSubsequenceCaseInsensitive = HasSubsequenceImpl<NameHasSubsequenceCaseInsensitive, HasSubsequenceCaseInsensitiveASCII>;
+}
+
+REGISTER_FUNCTION(hasSubsequenceCaseInsensitive)
+{
+    factory.registerFunction<FunctionHasSubsequenceCaseInsensitive>({}, FunctionFactory::CaseInsensitive);
+}
+
+}

src/Functions/hasSubsequenceCaseInsensitiveUTF8.cpp

@@ -0,0 +1,31 @@
+#include <Functions/FunctionFactory.h>
+#include <Functions/HasSubsequenceImpl.h>
+
+#include "Poco/Unicode.h"
+
+namespace DB
+{
+namespace
+{
+
+struct HasSubsequenceCaseInsensitiveUTF8
+{
+    static constexpr bool is_utf8 = true;
+
+    static int toLowerIfNeed(int code_point) { return Poco::Unicode::toLower(code_point); }
+};
+
+struct NameHasSubsequenceCaseInsensitiveUTF8
+{
+    static constexpr auto name = "hasSubsequenceCaseInsensitiveUTF8";
+};
+
+using FunctionHasSubsequenceCaseInsensitiveUTF8 = HasSubsequenceImpl<NameHasSubsequenceCaseInsensitiveUTF8, HasSubsequenceCaseInsensitiveUTF8>;
+}
+
+REGISTER_FUNCTION(hasSubsequenceCaseInsensitiveUTF8)
+{
+    factory.registerFunction<FunctionHasSubsequenceCaseInsensitiveUTF8>({}, FunctionFactory::CaseInsensitive);
+}
+
+}

src/Functions/hasSubsequenceUTF8.cpp

@@ -0,0 +1,30 @@
+#include <Functions/FunctionFactory.h>
+#include <Functions/HasSubsequenceImpl.h>
+
+
+namespace DB
+{
+namespace
+{
+
+struct HasSubsequenceCaseSensitiveUTF8
+{
+    static constexpr bool is_utf8 = true;
+
+    static int toLowerIfNeed(int code_point) { return code_point; }
+};
+
+struct NameHasSubsequenceUTF8
+{
+    static constexpr auto name = "hasSubsequenceUTF8";
+};
+
+using FunctionHasSubsequenceUTF8 = HasSubsequenceImpl<NameHasSubsequenceUTF8, HasSubsequenceCaseSensitiveUTF8>;
+}
+
+REGISTER_FUNCTION(hasSubsequenceUTF8)
+{
+    factory.registerFunction<FunctionHasSubsequenceUTF8>({}, FunctionFactory::CaseInsensitive);
+}
+
+}

src/Functions/like.cpp

@@ -1,4 +1,3 @@
-#include "FunctionsStringSearch.h"
 #include "FunctionFactory.h"
 #include "like.h"
 

src/IO/WriteHelpers.h

@@ -905,26 +905,26 @@ inline void writeText(const IPv4 & x, WriteBuffer & buf) { writeIPv4Text(x, buf)
 inline void writeText(const IPv6 & x, WriteBuffer & buf) { writeIPv6Text(x, buf); }
 
 template <typename T>
-void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr, bool trailing_zeros,
-                            bool fixed_fractional_length, UInt32 fractional_length)
+void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr, bool trailing_zeros)
 {
     /// If it's big integer, but the number of digits is small,
     /// use the implementation for smaller integers for more efficient arithmetic.
+
     if constexpr (std::is_same_v<T, Int256>)
     {
         if (x <= std::numeric_limits<UInt32>::max())
         {
-            writeDecimalFractional(static_cast<UInt32>(x), scale, ostr, trailing_zeros, fixed_fractional_length, fractional_length);
+            writeDecimalFractional(static_cast<UInt32>(x), scale, ostr, trailing_zeros);
             return;
         }
         else if (x <= std::numeric_limits<UInt64>::max())
         {
-            writeDecimalFractional(static_cast<UInt64>(x), scale, ostr, trailing_zeros, fixed_fractional_length, fractional_length);
+            writeDecimalFractional(static_cast<UInt64>(x), scale, ostr, trailing_zeros);
             return;
         }
         else if (x <= std::numeric_limits<UInt128>::max())
         {
-            writeDecimalFractional(static_cast<UInt128>(x), scale, ostr, trailing_zeros, fixed_fractional_length, fractional_length);
+            writeDecimalFractional(static_cast<UInt128>(x), scale, ostr, trailing_zeros);
             return;
         }
     }
@@ -932,36 +932,24 @@ void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr, bool
     {
         if (x <= std::numeric_limits<UInt32>::max())
         {
-            writeDecimalFractional(static_cast<UInt32>(x), scale, ostr, trailing_zeros, fixed_fractional_length, fractional_length);
+            writeDecimalFractional(static_cast<UInt32>(x), scale, ostr, trailing_zeros);
             return;
         }
         else if (x <= std::numeric_limits<UInt64>::max())
         {
-            writeDecimalFractional(static_cast<UInt64>(x), scale, ostr, trailing_zeros, fixed_fractional_length, fractional_length);
+            writeDecimalFractional(static_cast<UInt64>(x), scale, ostr, trailing_zeros);
             return;
         }
     }
 
     constexpr size_t max_digits = std::numeric_limits<UInt256>::digits10;
     assert(scale <= max_digits);
-    assert(fractional_length <= max_digits);
-
     char buf[max_digits];
-    memset(buf, '0', std::max(scale, fractional_length));
+    memset(buf, '0', scale);
 
     T value = x;
     Int32 last_nonzero_pos = 0;
-
-    if (fixed_fractional_length && fractional_length < scale)
-    {
-        T new_value = value / DecimalUtils::scaleMultiplier<Int256>(scale - fractional_length - 1);
-        auto round_carry = new_value % 10;
-        value = new_value / 10;
-        if (round_carry >= 5)
-            value += 1;
-    }
-
-    for (Int32 pos = fixed_fractional_length ? std::min(scale - 1, fractional_length - 1) : scale - 1; pos >= 0; --pos)
+    for (Int32 pos = scale - 1; pos >= 0; --pos)
     {
         auto remainder = value % 10;
         value /= 10;
@@ -973,12 +961,11 @@ void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr, bool
     }
 
     writeChar('.', ostr);
-    ostr.write(buf, fixed_fractional_length ? fractional_length : (trailing_zeros ? scale : last_nonzero_pos + 1));
+    ostr.write(buf, trailing_zeros ? scale : last_nonzero_pos + 1);
 }
 
 template <typename T>
-void writeText(Decimal<T> x, UInt32 scale, WriteBuffer & ostr, bool trailing_zeros,
-               bool fixed_fractional_length = false, UInt32 fractional_length = 0)
+void writeText(Decimal<T> x, UInt32 scale, WriteBuffer & ostr, bool trailing_zeros)
 {
     T part = DecimalUtils::getWholePart(x, scale);
 
@@ -989,7 +976,7 @@ void writeText(Decimal<T> x, UInt32 scale, WriteBuffer & ostr, bool trailing_zer
 
     writeIntText(part, ostr);
 
-    if (scale || (fixed_fractional_length && fractional_length > 0))
+    if (scale)
     {
         part = DecimalUtils::getFractionalPart(x, scale);
         if (part || trailing_zeros)
@@ -997,7 +984,7 @@ void writeText(Decimal<T> x, UInt32 scale, WriteBuffer & ostr, bool trailing_zer
             if (part < 0)
                 part *= T(-1);
 
-            writeDecimalFractional(part, scale, ostr, trailing_zeros, fixed_fractional_length, fractional_length);
+            writeDecimalFractional(part, scale, ostr, trailing_zeros);
         }
     }
 }

src/Interpreters/ActionsDAG.cpp

@@ -2515,11 +2515,21 @@ FindOriginalNodeForOutputName::FindOriginalNodeForOutputName(const ActionsDAGPtr
         /// find input node which refers to the output node
         /// consider only aliases on the path
         const auto * node = output_node;
-        while (node && node->type == ActionsDAG::ActionType::ALIAS)
+        while (node)
         {
-            /// alias has only one child
-            chassert(node->children.size() == 1);
-            node = node->children.front();
+            if (node->type == ActionsDAG::ActionType::ALIAS)
+            {
+                node = node->children.front();
+            }
+            /// materiailze() function can occur when dealing with views
+            /// TODO: not sure if it should be done here, looks too generic place
+            else if (node->type == ActionsDAG::ActionType::FUNCTION && node->function_base->getName() == "materialize")
+            {
+                chassert(node->children.size() == 1);
+                node = node->children.front();
+            }
+            else
+                break;
         }
         if (node && node->type == ActionsDAG::ActionType::INPUT)
             index.emplace(output_node->result_name, node);

src/Interpreters/Cache/LRUFileCachePriority.h

@@ -5,6 +5,11 @@
 #include <Interpreters/Cache/FileCacheKey.h>
 #include <Common/logger_useful.h>
 
+namespace CurrentMetrics
+{
+    extern const Metric FilesystemCacheSizeLimit;
+}
+
 namespace DB
 {
 
@@ -18,7 +23,10 @@ private:
     using LRUQueueIterator = typename LRUQueue::iterator;
 
 public:
-    LRUFileCachePriority(size_t max_size_, size_t max_elements_) : IFileCachePriority(max_size_, max_elements_) {}
+    LRUFileCachePriority(size_t max_size_, size_t max_elements_) : IFileCachePriority(max_size_, max_elements_)
+    {
+        CurrentMetrics::set(CurrentMetrics::FilesystemCacheSizeLimit, max_size_);
+    }
 
     size_t getSize(const CacheGuard::Lock &) const override { return current_size; }
 

src/Interpreters/ExpressionAnalyzer.cpp

@@ -548,15 +548,17 @@ void ExpressionAnalyzer::getRootActionsForWindowFunctions(const ASTPtr & ast, bo
 
 void ExpressionAnalyzer::makeAggregateDescriptions(ActionsDAGPtr & actions, AggregateDescriptions & descriptions)
 {
-    for (const ASTFunction * node : aggregates())
+    for (const ASTPtr & ast : aggregates())
     {
+        const ASTFunction & node = typeid_cast<const ASTFunction &>(*ast);
+
         AggregateDescription aggregate;
-        if (node->arguments)
-            getRootActionsNoMakeSet(node->arguments, actions);
+        if (node.arguments)
+            getRootActionsNoMakeSet(node.arguments, actions);
 
-        aggregate.column_name = node->getColumnName();
+        aggregate.column_name = node.getColumnName();
 
-        const ASTs & arguments = node->arguments ? node->arguments->children : ASTs();
+        const ASTs & arguments = node.arguments ? node.arguments->children : ASTs();
         aggregate.argument_names.resize(arguments.size());
         DataTypes types(arguments.size());
 
@@ -568,7 +570,7 @@ void ExpressionAnalyzer::makeAggregateDescriptions(ActionsDAGPtr & actions, Aggr
             {
                 throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
                     "Unknown identifier '{}' in aggregate function '{}'",
-                    name, node->formatForErrorMessage());
+                    name, node.formatForErrorMessage());
             }
 
             types[i] = dag_node->result_type;
@@ -576,8 +578,8 @@ void ExpressionAnalyzer::makeAggregateDescriptions(ActionsDAGPtr & actions, Aggr
         }
 
         AggregateFunctionProperties properties;
-        aggregate.parameters = (node->parameters) ? getAggregateFunctionParametersArray(node->parameters, "", getContext()) : Array();
-        aggregate.function = AggregateFunctionFactory::instance().get(node->name, types, aggregate.parameters, properties);
+        aggregate.parameters = (node.parameters) ? getAggregateFunctionParametersArray(node.parameters, "", getContext()) : Array();
+        aggregate.function = AggregateFunctionFactory::instance().get(node.name, types, aggregate.parameters, properties);
 
         descriptions.push_back(aggregate);
     }
@@ -744,12 +746,13 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions)
     }
 
     // Window functions
-    for (const ASTFunction * function_node : syntax->window_function_asts)
+    for (const ASTPtr & ast : syntax->window_function_asts)
     {
-        assert(function_node->is_window_function);
+        const ASTFunction & function_node = typeid_cast<const ASTFunction &>(*ast);
+        assert(function_node.is_window_function);
 
         WindowFunctionDescription window_function;
-        window_function.function_node = function_node;
+        window_function.function_node = &function_node;
         window_function.column_name
             = window_function.function_node->getColumnName();
         window_function.function_parameters
@@ -760,7 +763,7 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions)
 
         // Requiring a constant reference to a shared pointer to non-const AST
         // doesn't really look sane, but the visitor does indeed require it.
-        // Hence we clone the node (not very sane either, I know).
+        // Hence, we clone the node (not very sane either, I know).
         getRootActionsNoMakeSet(window_function.function_node->clone(), actions);
 
         const ASTs & arguments
@@ -793,22 +796,22 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions)
         // Find the window corresponding to this function. It may be either
         // referenced by name and previously defined in WINDOW clause, or it
         // may be defined inline.
-        if (!function_node->window_name.empty())
+        if (!function_node.window_name.empty())
         {
-            auto it = window_descriptions.find(function_node->window_name);
+            auto it = window_descriptions.find(function_node.window_name);
             if (it == std::end(window_descriptions))
             {
                 throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
                     "Window '{}' is not defined (referenced by '{}')",
-                    function_node->window_name,
-                    function_node->formatForErrorMessage());
+                    function_node.window_name,
+                    function_node.formatForErrorMessage());
             }
 
             it->second.window_functions.push_back(window_function);
         }
         else
         {
-            const auto & definition = function_node->window_definition->as<
+            const auto & definition = function_node.window_definition->as<
                 const ASTWindowDefinition &>();
             WindowDescription desc;
             desc.window_name = definition.getDefaultWindowName();
@@ -1323,10 +1326,13 @@ void SelectQueryExpressionAnalyzer::appendAggregateFunctionsArguments(Expression
         GetAggregatesVisitor(data).visit(select_query->orderBy());
 
     /// TODO: data.aggregates -> aggregates()
-    for (const ASTFunction * node : data.aggregates)
-        if (node->arguments)
-            for (auto & argument : node->arguments->children)
+    for (const ASTPtr & ast : data.aggregates)
+    {
+        const ASTFunction & node = typeid_cast<const ASTFunction &>(*ast);
+        if (node.arguments)
+            for (auto & argument : node.arguments->children)
                 getRootActions(argument, only_types, step.actions());
+    }
 }
 
 void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments(

src/Interpreters/ExpressionAnalyzer.h

@@ -168,7 +168,7 @@ protected:
     const ConstStoragePtr & storage() const { return syntax->storage; } /// The main table in FROM clause, if exists.
     const TableJoin & analyzedJoin() const { return *syntax->analyzed_join; }
     const NamesAndTypesList & sourceColumns() const { return syntax->required_source_columns; }
-    const std::vector<const ASTFunction *> & aggregates() const { return syntax->aggregates; }
+    const ASTs & aggregates() const { return syntax->aggregates; }
     /// Find global subqueries in the GLOBAL IN/JOIN sections. Fills in external_tables.
     void initGlobalSubqueriesAndExternalTables(bool do_global, bool is_explain);
 

src/Interpreters/GetAggregatesVisitor.h

@@ -26,8 +26,8 @@ public:
         // Explicit empty initializers are needed to make designated initializers
         // work on GCC 10.
         std::unordered_set<String> uniq_names {};
-        std::vector<const ASTFunction *> aggregates {};
-        std::vector<const ASTFunction *> window_functions {};
+        ASTs aggregates;
+        ASTs window_functions;
     };
 
     static bool needChildVisit(const ASTPtr & node, const ASTPtr & child)
@@ -61,7 +61,7 @@ public:
     }
 
 private:
-    static void visit(const ASTFunction & node, const ASTPtr &, Data & data)
+    static void visit(const ASTFunction & node, const ASTPtr & ast, Data & data)
     {
         if (isAggregateFunction(node))
         {
@@ -74,7 +74,7 @@ private:
                 return;
 
             data.uniq_names.insert(column_name);
-            data.aggregates.push_back(&node);
+            data.aggregates.push_back(ast);
         }
         else if (node.is_window_function)
         {
@@ -87,7 +87,7 @@ private:
                 return;
 
             data.uniq_names.insert(column_name);
-            data.window_functions.push_back(&node);
+            data.window_functions.push_back(ast);
         }
     }
 

src/Interpreters/InterpreterSetQuery.cpp

@@ -65,6 +65,9 @@ void InterpreterSetQuery::applySettingsFromQuery(const ASTPtr & ast, ContextMuta
     }
     else if (const auto * explain_query = ast->as<ASTExplainQuery>())
     {
+        if (explain_query->settings_ast)
+            InterpreterSetQuery(explain_query->settings_ast, context_).executeForCurrentContext();
+
         applySettingsFromQuery(explain_query->getExplainedQuery(), context_);
     }
     else if (const auto * query_with_output = dynamic_cast<const ASTQueryWithOutput *>(ast.get()))

src/Interpreters/ProcessList.cpp

@@ -223,10 +223,7 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q
             {
                 /// Set up memory profiling
                 thread_group->memory_tracker.setProfilerStep(settings.memory_profiler_step);
-
                 thread_group->memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability);
-                thread_group->memory_tracker.setSampleMinAllocationSize(settings.memory_profiler_sample_min_allocation_size);
-                thread_group->memory_tracker.setSampleMaxAllocationSize(settings.memory_profiler_sample_max_allocation_size);
                 thread_group->performance_counters.setTraceProfileEvents(settings.trace_profile_events);
             }
 

src/Interpreters/ThreadStatusExt.cpp

@@ -83,8 +83,6 @@ ThreadGroupPtr ThreadGroup::createForBackgroundProcess(ContextPtr storage_contex
     const Settings & settings = storage_context->getSettingsRef();
     group->memory_tracker.setProfilerStep(settings.memory_profiler_step);
     group->memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability);
-    group->memory_tracker.setSampleMinAllocationSize(settings.memory_profiler_sample_min_allocation_size);
-    group->memory_tracker.setSampleMaxAllocationSize(settings.memory_profiler_sample_max_allocation_size);
     group->memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator);
     group->memory_tracker.setParent(&background_memory_tracker);
     if (settings.memory_tracker_fault_probability > 0.0)

src/Interpreters/TreeRewriter.cpp

@@ -731,7 +731,7 @@ void expandGroupByAll(ASTSelectQuery * select_query)
     select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, group_expression_list);
 }
 
-std::vector<const ASTFunction *> getAggregates(ASTPtr & query, const ASTSelectQuery & select_query)
+ASTs getAggregates(ASTPtr & query, const ASTSelectQuery & select_query)
 {
     /// There can not be aggregate functions inside the WHERE and PREWHERE.
     if (select_query.where())
@@ -743,11 +743,12 @@ std::vector<const ASTFunction *> getAggregates(ASTPtr & query, const ASTSelectQu
     GetAggregatesVisitor(data).visit(query);
 
     /// There can not be other aggregate functions within the aggregate functions.
-    for (const ASTFunction * node : data.aggregates)
+    for (const ASTPtr & ast : data.aggregates)
     {
-        if (node->arguments)
+        const ASTFunction & node = typeid_cast<const ASTFunction &>(*ast);
+        if (node.arguments)
         {
-            for (auto & arg : node->arguments->children)
+            for (auto & arg : node.arguments->children)
             {
                 assertNoAggregates(arg, "inside another aggregate function");
                 // We also can't have window functions inside aggregate functions,
@@ -759,7 +760,7 @@ std::vector<const ASTFunction *> getAggregates(ASTPtr & query, const ASTSelectQu
     return data.aggregates;
 }
 
-std::vector<const ASTFunction *> getWindowFunctions(ASTPtr & query, const ASTSelectQuery & select_query)
+ASTs getWindowFunctions(ASTPtr & query, const ASTSelectQuery & select_query)
 {
     /// There can not be window functions inside the WHERE, PREWHERE and HAVING
     if (select_query.having())
@@ -777,20 +778,16 @@ std::vector<const ASTFunction *> getWindowFunctions(ASTPtr & query, const ASTSel
     /// Window functions cannot be inside aggregates or other window functions.
     /// Aggregate functions can be inside window functions because they are
     /// calculated earlier.
-    for (const ASTFunction * node : data.window_functions)
+    for (const ASTPtr & ast : data.window_functions)
     {
-        if (node->arguments)
-        {
-            for (auto & arg : node->arguments->children)
-            {
-                assertNoWindows(arg, "inside another window function");
-            }
-        }
+        const ASTFunction & node = typeid_cast<const ASTFunction &>(*ast);
 
-        if (node->window_definition)
-        {
-            assertNoWindows(node->window_definition, "inside window definition");
-        }
+        if (node.arguments)
+            for (auto & arg : node.arguments->children)
+                assertNoWindows(arg, "inside another window function");
+
+        if (node.window_definition)
+            assertNoWindows(node.window_definition, "inside window definition");
     }
 
     return data.window_functions;
@@ -1357,8 +1354,8 @@ TreeRewriterResultPtr TreeRewriter::analyze(
         GetAggregatesVisitor(data).visit(query);
 
         /// There can not be other aggregate functions within the aggregate functions.
-        for (const ASTFunction * node : data.aggregates)
-            for (auto & arg : node->arguments->children)
+        for (const ASTPtr & node : data.aggregates)
+            for (auto & arg : typeid_cast<const ASTFunction &>(*node).arguments->children)
                 assertNoAggregates(arg, "inside another aggregate function");
         result.aggregates = data.aggregates;
     }

src/Interpreters/TreeRewriter.h

@@ -41,8 +41,8 @@ struct TreeRewriterResult
 
     Aliases aliases;
 
-    std::vector<const ASTFunction *> aggregates;
-    std::vector<const ASTFunction *> window_function_asts;
+    ASTs aggregates;
+    ASTs window_function_asts;
     ASTs expressions_with_window_function;
 
     /// Which column is needed to be ARRAY-JOIN'ed to get the specified.

src/Interpreters/tests/gtest_lru_file_cache.cpp

@@ -470,6 +470,7 @@ TEST_F(FileCacheTest, get)
 
                 auto & file_segment2 = get(holder2, 2);
                 ASSERT_TRUE(file_segment2.getOrSetDownloader() != FileSegment::getCallerId());
+                ASSERT_EQ(file_segment2.state(), State::DOWNLOADING);
 
                 {
                     std::lock_guard lock(mutex);
@@ -478,8 +479,7 @@ TEST_F(FileCacheTest, get)
                 cv.notify_one();
 
                 file_segment2.wait(file_segment2.range().right);
-                file_segment2.complete();
-                ASSERT_TRUE(file_segment2.state() == State::DOWNLOADED);
+                ASSERT_EQ(file_segment2.getDownloadedSize(false), file_segment2.range().size());
             });
 
             {
@@ -488,7 +488,7 @@ TEST_F(FileCacheTest, get)
             }
 
             download(file_segment);
-            ASSERT_TRUE(file_segment.state() == State::DOWNLOADED);
+            ASSERT_EQ(file_segment.state(), State::DOWNLOADED);
 
             other_1.join();
 

src/Parsers/ParserDescribeTableQuery.cpp

@@ -3,6 +3,7 @@
 #include <Parsers/CommonParsers.h>
 #include <Parsers/ParserDescribeTableQuery.h>
 #include <Parsers/ParserTablesInSelectQuery.h>
+#include <Parsers/ParserSetQuery.h>
 
 #include <Common/typeid_cast.h>
 
@@ -16,8 +17,10 @@ bool ParserDescribeTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & ex
     ParserKeyword s_describe("DESCRIBE");
     ParserKeyword s_desc("DESC");
     ParserKeyword s_table("TABLE");
+    ParserKeyword s_settings("SETTINGS");
     ParserToken s_dot(TokenType::Dot);
     ParserIdentifier name_p;
+    ParserSetQuery parser_settings(true);
 
     ASTPtr database;
     ASTPtr table;
@@ -29,12 +32,21 @@ bool ParserDescribeTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & ex
 
     s_table.ignore(pos, expected);
 
-    ASTPtr table_expression;
-    if (!ParserTableExpression().parse(pos, table_expression, expected))
+    if (!ParserTableExpression().parse(pos, query->table_expression, expected))
         return false;
 
-    query->children.push_back(std::move(table_expression));
-    query->table_expression = query->children.back();
+    /// For compatibility with SELECTs, where SETTINGS can be in front of FORMAT
+    ASTPtr settings;
+    if (s_settings.ignore(pos, expected))
+    {
+        if (!parser_settings.parse(pos, query->settings_ast, expected))
+            return false;
+    }
+
+    query->children.push_back(query->table_expression);
+
+    if (query->settings_ast)
+        query->children.push_back(query->settings_ast);
 
     node = query;
 

src/Parsers/ParserQueryWithOutput.cpp

@@ -156,7 +156,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
 
     // SETTINGS key1 = value1, key2 = value2, ...
     ParserKeyword s_settings("SETTINGS");
-    if (s_settings.ignore(pos, expected))
+    if (!query_with_output.settings_ast && s_settings.ignore(pos, expected))
     {
         ParserSetQuery parser_settings(true);
         if (!parser_settings.parse(pos, query_with_output.settings_ast, expected))

src/Parsers/ParserTablePropertiesQuery.cpp

@@ -14,8 +14,6 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
 {
     ParserKeyword s_exists("EXISTS");
     ParserKeyword s_temporary("TEMPORARY");
-    ParserKeyword s_describe("DESCRIBE");
-    ParserKeyword s_desc("DESC");
     ParserKeyword s_show("SHOW");
     ParserKeyword s_create("CREATE");
     ParserKeyword s_database("DATABASE");

src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp

@@ -92,6 +92,10 @@ bool optimizeUseNormalProjections(Stack & stack, QueryPlan::Nodes & nodes)
             break;
     }
 
+    /// Dangling query plan node. This might be generated by StorageMerge.
+    if (iter->node->step.get() == reading)
+        return false;
+
     const auto metadata = reading->getStorageMetadata();
     const auto & projections = metadata->projections;
 
@@ -105,8 +109,8 @@ bool optimizeUseNormalProjections(Stack & stack, QueryPlan::Nodes & nodes)
 
     QueryDAG query;
     {
-        auto & clild = iter->node->children[iter->next_child - 1];
-        if (!query.build(*clild))
+        auto & child = iter->node->children[iter->next_child - 1];
+        if (!query.build(*child))
             return false;
 
         if (query.dag)

src/Storages/MergeTree/MergeTreeData.cpp

@@ -382,7 +382,7 @@ MergeTreeData::MergeTreeData(
     checkTTLExpressions(metadata_, metadata_);
 
     String reason;
-    if (!canUsePolymorphicParts(*settings, &reason) && !reason.empty())
+    if (!canUsePolymorphicParts(*settings, reason) && !reason.empty())
         LOG_WARNING(log, "{} Settings 'min_rows_for_wide_part'and 'min_bytes_for_wide_part' will be ignored.", reason);
 
 #if !USE_ROCKSDB
@@ -3323,7 +3323,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
                 MergeTreeSettings copy = *getSettings();
                 copy.applyChange(changed_setting);
                 String reason;
-                if (!canUsePolymorphicParts(copy, &reason) && !reason.empty())
+                if (!canUsePolymorphicParts(copy, reason) && !reason.empty())
                     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Can't change settings. Reason: {}", reason);
             }
 
@@ -3348,7 +3348,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
                 auto copy = getDefaultSettings();
                 copy->applyChanges(new_changes);
                 String reason;
-                if (!canUsePolymorphicParts(*copy, &reason) && !reason.empty())
+                if (!canUsePolymorphicParts(*copy, reason) && !reason.empty())
                     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Can't change settings. Reason: {}", reason);
             }
 
@@ -3390,8 +3390,9 @@ MergeTreeDataPartFormat MergeTreeData::choosePartFormat(size_t bytes_uncompresse
     using PartType = MergeTreeDataPartType;
     using PartStorageType = MergeTreeDataPartStorageType;
 
-     const auto settings = getSettings();
-     if (!canUsePolymorphicParts(*settings))
+    String out_reason;
+    const auto settings = getSettings();
+    if (!canUsePolymorphicParts(*settings, out_reason))
         return {PartType::Wide, PartStorageType::Full};
 
     auto satisfies = [&](const auto & min_bytes_for, const auto & min_rows_for)
@@ -8010,22 +8011,23 @@ bool MergeTreeData::partsContainSameProjections(const DataPartPtr & left, const
 
 bool MergeTreeData::canUsePolymorphicParts() const
 {
-    return canUsePolymorphicParts(*getSettings(), nullptr);
+    String unused;
+    return canUsePolymorphicParts(*getSettings(), unused);
 }
 
-bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, String * out_reason) const
+bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, String & out_reason) const
 {
     if (!canUseAdaptiveGranularity())
     {
-        if (out_reason && (settings.min_rows_for_wide_part != 0 || settings.min_bytes_for_wide_part != 0
+        if ((settings.min_rows_for_wide_part != 0 || settings.min_bytes_for_wide_part != 0
             || settings.min_rows_for_compact_part != 0 || settings.min_bytes_for_compact_part != 0))
         {
-            *out_reason = fmt::format(
-                    "Table can't create parts with adaptive granularity, but settings"
-                    " min_rows_for_wide_part = {}"
-                    ", min_bytes_for_wide_part = {}"
-                    ". Parts with non-adaptive granularity can be stored only in Wide (default) format.",
-                    settings.min_rows_for_wide_part, settings.min_bytes_for_wide_part);
+            out_reason = fmt::format(
+                "Table can't create parts with adaptive granularity, but settings"
+                " min_rows_for_wide_part = {}"
+                ", min_bytes_for_wide_part = {}"
+                ". Parts with non-adaptive granularity can be stored only in Wide (default) format.",
+                settings.min_rows_for_wide_part, settings.min_bytes_for_wide_part);
         }
 
         return false;

src/Storages/MergeTree/MergeTreeData.h

@@ -1484,7 +1484,7 @@ private:
     /// Check selected parts for movements. Used by ALTER ... MOVE queries.
     CurrentlyMovingPartsTaggerPtr checkPartsForMove(const DataPartsVector & parts, SpacePtr space);
 
-    bool canUsePolymorphicParts(const MergeTreeSettings & settings, String * out_reason = nullptr) const;
+    bool canUsePolymorphicParts(const MergeTreeSettings & settings, String & out_reason) const;
 
     std::mutex write_ahead_log_mutex;
     WriteAheadLogPtr write_ahead_log;

src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp

@@ -136,7 +136,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge(
     const AllowedMergingPredicate & can_merge_callback,
     bool merge_with_ttl_allowed,
     const MergeTreeTransactionPtr & txn,
-    String * out_disable_reason,
+    String & out_disable_reason,
     const PartitionIdsHint * partitions_hint)
 {
     MergeTreeData::DataPartsVector data_parts = getDataPartsToSelectMergeFrom(txn, partitions_hint);
@@ -145,8 +145,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge(
 
     if (data_parts.empty())
     {
-        if (out_disable_reason)
-            *out_disable_reason = "There are no parts in the table";
+        out_disable_reason = "There are no parts in the table";
         return SelectPartsDecision::CANNOT_SELECT;
     }
 
@@ -154,8 +153,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge(
 
     if (info.parts_selected_precondition == 0)
     {
-        if (out_disable_reason)
-            *out_disable_reason = "No parts satisfy preconditions for merge";
+        out_disable_reason = "No parts satisfy preconditions for merge";
         return SelectPartsDecision::CANNOT_SELECT;
     }
 
@@ -179,8 +177,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge(
                 /*optimize_skip_merged_partitions=*/true);
     }
 
-    if (out_disable_reason)
-        *out_disable_reason = "There is no need to merge parts according to merge selector algorithm";
+    out_disable_reason = "There is no need to merge parts according to merge selector algorithm";
     return SelectPartsDecision::CANNOT_SELECT;
 }
 
@@ -197,7 +194,8 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart
 
     auto metadata_snapshot = data.getInMemoryMetadataPtr();
 
-    MergeSelectingInfo info = getPossibleMergeRanges(data_parts, can_merge_callback, txn);
+    String out_reason;
+    MergeSelectingInfo info = getPossibleMergeRanges(data_parts, can_merge_callback, txn, out_reason);
 
     if (info.parts_selected_precondition == 0)
         return res;
@@ -227,7 +225,7 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart
         /// This method should have been const, but something went wrong... it's const with dry_run = true
         auto status = const_cast<MergeTreeDataMergerMutator *>(this)->selectPartsToMergeFromRanges(
                 future_part, /*aggressive*/ false, max_total_size_to_merge, merge_with_ttl_allowed,
-                metadata_snapshot, ranges_per_partition[i], info.current_time, &out_disable_reason,
+                metadata_snapshot, ranges_per_partition[i], info.current_time, out_disable_reason,
                 /* dry_run */ true);
         if (status == SelectPartsDecision::SELECTED)
             res.insert(all_partition_ids[i]);
@@ -331,7 +329,7 @@ MergeTreeDataMergerMutator::MergeSelectingInfo MergeTreeDataMergerMutator::getPo
     const MergeTreeData::DataPartsVector & data_parts,
     const AllowedMergingPredicate & can_merge_callback,
     const MergeTreeTransactionPtr & txn,
-    String * out_disable_reason) const
+    String & out_disable_reason) const
 {
     MergeSelectingInfo res;
 
@@ -444,7 +442,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
     const StorageMetadataPtr & metadata_snapshot,
     const IMergeSelector::PartsRanges & parts_ranges,
     const time_t & current_time,
-    String * out_disable_reason,
+    String & out_disable_reason,
     bool dry_run)
 {
     const auto data_settings = data.getSettings();
@@ -515,8 +513,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
 
         if (parts_to_merge.empty())
         {
-            if (out_disable_reason)
-                *out_disable_reason = "Did not find any parts to merge (with usual merge selectors)";
+            out_disable_reason = "Did not find any parts to merge (with usual merge selectors)";
             return SelectPartsDecision::CANNOT_SELECT;
         }
     }
@@ -563,22 +560,20 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinParti
     bool final,
     const StorageMetadataPtr & metadata_snapshot,
     const MergeTreeTransactionPtr & txn,
-    String * out_disable_reason,
+    String & out_disable_reason,
     bool optimize_skip_merged_partitions)
 {
     MergeTreeData::DataPartsVector parts = selectAllPartsFromPartition(partition_id);
 
     if (parts.empty())
     {
-        if (out_disable_reason)
-            *out_disable_reason = "There are no parts inside partition";
+        out_disable_reason = "There are no parts inside partition";
         return SelectPartsDecision::CANNOT_SELECT;
     }
 
     if (!final && parts.size() == 1)
     {
-        if (out_disable_reason)
-            *out_disable_reason = "There is only one part inside partition";
+        out_disable_reason = "There is only one part inside partition";
         return SelectPartsDecision::CANNOT_SELECT;
     }
 
@@ -587,8 +582,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinParti
     if (final && optimize_skip_merged_partitions && parts.size() == 1 && parts[0]->info.level > 0 &&
         (!metadata_snapshot->hasAnyTTL() || parts[0]->checkAllTTLCalculated(metadata_snapshot)))
     {
-        if (out_disable_reason)
-            *out_disable_reason = "Partition skipped due to optimize_skip_merged_partitions";
+        out_disable_reason = "Partition skipped due to optimize_skip_merged_partitions";
         return SelectPartsDecision::NOTHING_TO_MERGE;
     }
 
@@ -629,9 +623,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinParti
                 static_cast<int>((DISK_USAGE_COEFFICIENT_TO_SELECT - 1.0) * 100));
         }
 
-        if (out_disable_reason)
-            *out_disable_reason = fmt::format("Insufficient available disk space, required {}", ReadableSize(required_disk_space));
-
+        out_disable_reason = fmt::format("Insufficient available disk space, required {}", ReadableSize(required_disk_space));
         return SelectPartsDecision::CANNOT_SELECT;
     }
 

src/Storages/MergeTree/MergeTreeDataMergerMutator.h

@@ -43,7 +43,7 @@ public:
     using AllowedMergingPredicate = std::function<bool (const MergeTreeData::DataPartPtr &,
                                                         const MergeTreeData::DataPartPtr &,
                                                         const MergeTreeTransaction *,
-                                                        String *)>;
+                                                        String &)>;
 
     explicit MergeTreeDataMergerMutator(MergeTreeData & data_);
 
@@ -92,7 +92,7 @@ public:
         const MergeTreeData::DataPartsVector & data_parts,
         const AllowedMergingPredicate & can_merge_callback,
         const MergeTreeTransactionPtr & txn,
-        String * out_disable_reason = nullptr) const;
+        String & out_disable_reason) const;
 
     /// The third step of selecting parts to merge: takes ranges that we can merge, and selects parts that we want to merge
     SelectPartsDecision selectPartsToMergeFromRanges(
@@ -103,7 +103,7 @@ public:
         const StorageMetadataPtr & metadata_snapshot,
         const IMergeSelector::PartsRanges & parts_ranges,
         const time_t & current_time,
-        String * out_disable_reason = nullptr,
+        String & out_disable_reason,
         bool dry_run = false);
 
     String getBestPartitionToOptimizeEntire(const PartitionsInfo & partitions_info) const;
@@ -129,7 +129,7 @@ public:
         const AllowedMergingPredicate & can_merge,
         bool merge_with_ttl_allowed,
         const MergeTreeTransactionPtr & txn,
-        String * out_disable_reason = nullptr,
+        String & out_disable_reason,
         const PartitionIdsHint * partitions_hint = nullptr);
 
     /** Select all the parts in the specified partition for merge, if possible.
@@ -144,7 +144,7 @@ public:
         bool final,
         const StorageMetadataPtr & metadata_snapshot,
         const MergeTreeTransactionPtr & txn,
-        String * out_disable_reason = nullptr,
+        String & out_disable_reason,
         bool optimize_skip_merged_partitions = false);
 
     /** Creates a task to merge parts.

src/Storages/MergeTree/MergeTreeSettings.h

@@ -33,7 +33,7 @@ struct Settings;
     /** Data storing format settings. */ \
     M(UInt64, min_bytes_for_wide_part, 10485760, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \
     M(UInt64, min_rows_for_wide_part, 0, "Minimal number of rows to create part in wide format instead of compact", 0) \
-    M(Float, ratio_of_defaults_for_sparse_serialization, 1.0, "Minimal ratio of number of default values to number of all values in column to store it in sparse serializations. If >= 1, columns will be always written in full serialization.", 0) \
+    M(Float, ratio_of_defaults_for_sparse_serialization, 0.9375f, "Minimal ratio of number of default values to number of all values in column to store it in sparse serializations. If >= 1, columns will be always written in full serialization.", 0) \
     \
     /** Merge settings. */ \
     M(UInt64, merge_max_block_size, 8192, "How many rows in blocks should be formed for merge operations. By default has the same value as `index_granularity`.", 0) \
@@ -161,7 +161,7 @@ struct Settings;
     M(UInt64, min_marks_to_honor_max_concurrent_queries, 0, "Minimal number of marks to honor the MergeTree-level's max_concurrent_queries (0 - disabled). Queries will still be limited by other max_concurrent_queries settings.", 0) \
     M(UInt64, min_bytes_to_rebalance_partition_over_jbod, 0, "Minimal amount of bytes to enable part rebalance over JBOD array (0 - disabled).", 0) \
     M(Bool, check_sample_column_is_correct, true, "Check columns or columns by hash for sampling are unsigned integer.", 0) \
-    M(Bool, allow_vertical_merges_from_compact_to_wide_parts, false, "Allows vertical merges from compact to wide parts. This settings must have the same value on all replicas", 0) \
+    M(Bool, allow_vertical_merges_from_compact_to_wide_parts, true, "Allows vertical merges from compact to wide parts. This settings must have the same value on all replicas", 0) \
     M(Bool, enable_the_endpoint_id_with_zookeeper_name_prefix, false, "Enable the endpoint id with zookeeper name prefix for the replicated merge tree table", 0) \
     M(UInt64, zero_copy_merge_mutation_min_parts_size_sleep_before_lock, 1ULL * 1024 * 1024 * 1024, "If zero copy replication is enabled sleep random amount of time before trying to lock depending on parts size for merge or mutation", 0) \
     \
@@ -170,8 +170,9 @@ struct Settings;
     M(UInt64, part_moves_between_shards_delay_seconds, 30, "Time to wait before/after moving parts between shards.", 0) \
     M(Bool, use_metadata_cache, false, "Experimental feature to speed up parts loading process by using MergeTree metadata cache", 0) \
     M(Bool, allow_remote_fs_zero_copy_replication, false, "Don't use this setting in production, because it is not ready.", 0) \
-    M(String, remote_fs_zero_copy_zookeeper_path, "/clickhouse/zero_copy", "ZooKeeper path for Zero-copy table-independet info.", 0) \
+    M(String, remote_fs_zero_copy_zookeeper_path, "/clickhouse/zero_copy", "ZooKeeper path for zero-copy table-independent info.", 0) \
     M(Bool, remote_fs_zero_copy_path_compatible_mode, false, "Run zero-copy in compatible mode during conversion process.", 0) \
+    \
     /** Compress marks and primary key. */ \
     M(Bool, compress_marks, true, "Marks support compression, reduce mark file size and speed up network transmission.", 0) \
     M(Bool, compress_primary_key, true, "Primary key support compression, reduce primary key file size and speed up network transmission.", 0) \

src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp

@@ -2245,7 +2245,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::operator()(
     const MergeTreeData::DataPartPtr & left,
     const MergeTreeData::DataPartPtr & right,
     const MergeTreeTransaction *,
-    String * out_reason) const
+    String & out_reason) const
 {
     if (left)
         return canMergeTwoParts(left, right, out_reason);
@@ -2257,7 +2257,7 @@ template<typename VirtualPartsT, typename MutationsStateT>
 bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
     const MergeTreeData::DataPartPtr & left,
     const MergeTreeData::DataPartPtr & right,
-    String * out_reason) const
+    String & out_reason) const
 {
     /// A sketch of a proof of why this method actually works:
     ///
@@ -2301,22 +2301,19 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
     {
         if (pinned_part_uuids_ && pinned_part_uuids_->part_uuids.contains(part->uuid))
         {
-            if (out_reason)
-                *out_reason = "Part " + part->name + " has uuid " + toString(part->uuid) + " which is currently pinned";
+            out_reason = "Part " + part->name + " has uuid " + toString(part->uuid) + " which is currently pinned";
             return false;
         }
 
         if (inprogress_quorum_part_ && part->name == *inprogress_quorum_part_)
         {
-            if (out_reason)
-                *out_reason = "Quorum insert for part " + part->name + " is currently in progress";
+            out_reason = "Quorum insert for part " + part->name + " is currently in progress";
             return false;
         }
 
         if (prev_virtual_parts_ && prev_virtual_parts_->getContainingPart(part->info).empty())
         {
-            if (out_reason)
-                *out_reason = "Entry for part " + part->name + " hasn't been read from the replication log yet";
+            out_reason = "Entry for part " + part->name + " hasn't been read from the replication log yet";
             return false;
         }
     }
@@ -2330,8 +2327,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
     {
         if (partition_ids_hint && !partition_ids_hint->contains(left->info.partition_id))
         {
-            if (out_reason)
-                *out_reason = fmt::format("Uncommitted block were not loaded for unexpected partition {}", left->info.partition_id);
+            out_reason = fmt::format("Uncommitted block were not loaded for unexpected partition {}", left->info.partition_id);
             return false;
         }
 
@@ -2343,10 +2339,8 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
             auto block_it = block_numbers.upper_bound(left_max_block);
             if (block_it != block_numbers.end() && *block_it < right_min_block)
             {
-                if (out_reason)
-                    *out_reason = "Block number " + toString(*block_it) + " is still being inserted between parts "
-                        + left->name + " and " + right->name;
-
+                out_reason = "Block number " + toString(*block_it) + " is still being inserted between parts "
+                    + left->name + " and " + right->name;
                 return false;
             }
         }
@@ -2365,8 +2359,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
             String containing_part = virtual_parts_->getContainingPart(part->info);
             if (containing_part != part->name)
             {
-                if (out_reason)
-                    *out_reason = "Part " + part->name + " has already been assigned a merge into " + containing_part;
+                out_reason = "Part " + part->name + " has already been assigned a merge into " + containing_part;
                 return false;
             }
         }
@@ -2383,10 +2376,9 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
             Strings covered = virtual_parts_->getPartsCoveredBy(gap_part_info);
             if (!covered.empty())
             {
-                if (out_reason)
-                    *out_reason = "There are " + toString(covered.size()) + " parts (from " + covered.front()
-                        + " to " + covered.back() + ") that are still not present or being processed by "
-                        + " other background process on this replica between " + left->name + " and " + right->name;
+                out_reason = "There are " + toString(covered.size()) + " parts (from " + covered.front()
+                    + " to " + covered.back() + ") that are still not present or being processed by "
+                    + " other background process on this replica between " + left->name + " and " + right->name;
                 return false;
             }
         }
@@ -2402,9 +2394,8 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
 
         if (left_mutation_ver != right_mutation_ver)
         {
-            if (out_reason)
-                *out_reason = "Current mutation versions of parts " + left->name + " and " + right->name + " differ: "
-                    + toString(left_mutation_ver) + " and " + toString(right_mutation_ver) + " respectively";
+            out_reason = "Current mutation versions of parts " + left->name + " and " + right->name + " differ: "
+                + toString(left_mutation_ver) + " and " + toString(right_mutation_ver) + " respectively";
             return false;
         }
     }
@@ -2415,26 +2406,23 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
 template<typename VirtualPartsT, typename MutationsStateT>
 bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeSinglePart(
     const MergeTreeData::DataPartPtr & part,
-    String * out_reason) const
+    String & out_reason) const
 {
     if (pinned_part_uuids_ && pinned_part_uuids_->part_uuids.contains(part->uuid))
     {
-        if (out_reason)
-            *out_reason = fmt::format("Part {} has uuid {} which is currently pinned", part->name, part->uuid);
+        out_reason = fmt::format("Part {} has uuid {} which is currently pinned", part->name, part->uuid);
         return false;
     }
 
     if (inprogress_quorum_part_ && part->name == *inprogress_quorum_part_)
     {
-        if (out_reason)
-            *out_reason = fmt::format("Quorum insert for part {} is currently in progress", part->name);
+        out_reason = fmt::format("Quorum insert for part {} is currently in progress", part->name);
         return false;
     }
 
     if (prev_virtual_parts_ && prev_virtual_parts_->getContainingPart(part->info).empty())
     {
-        if (out_reason)
-            *out_reason = fmt::format("Entry for part {} hasn't been read from the replication log yet", part->name);
+        out_reason = fmt::format("Entry for part {} hasn't been read from the replication log yet", part->name);
         return false;
     }
 
@@ -2449,8 +2437,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeSinglePart(
         String containing_part = virtual_parts_->getContainingPart(part->info);
         if (containing_part != part->name)
         {
-            if (out_reason)
-                *out_reason = fmt::format("Part {} has already been assigned a merge into {}", part->name, containing_part);
+            out_reason = fmt::format("Part {} has already been assigned a merge into {}", part->name, containing_part);
             return false;
         }
     }
@@ -2459,7 +2446,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeSinglePart(
 }
 
 
-bool ReplicatedMergeTreeMergePredicate::partParticipatesInReplaceRange(const MergeTreeData::DataPartPtr & part, String * out_reason) const
+bool ReplicatedMergeTreeMergePredicate::partParticipatesInReplaceRange(const MergeTreeData::DataPartPtr & part, String & out_reason) const
 {
     std::lock_guard lock(queue.state_mutex);
     for (const auto & entry : queue.queue)
@@ -2472,9 +2459,7 @@ bool ReplicatedMergeTreeMergePredicate::partParticipatesInReplaceRange(const Mer
             if (part->info.isDisjoint(MergeTreePartInfo::fromPartName(part_name, queue.format_version)))
                 continue;
 
-            if (out_reason)
-                *out_reason = fmt::format("Part {} participates in REPLACE_RANGE {} ({})", part_name, entry->new_part_name, entry->znode_name);
-
+            out_reason = fmt::format("Part {} participates in REPLACE_RANGE {} ({})", part_name, entry->new_part_name, entry->znode_name);
             return true;
         }
     }

src/Storages/MergeTree/ReplicatedMergeTreeQueue.h

@@ -505,19 +505,19 @@ public:
     bool operator()(const MergeTreeData::DataPartPtr & left,
                     const MergeTreeData::DataPartPtr & right,
                     const MergeTreeTransaction * txn,
-                    String * out_reason = nullptr) const;
+                    String & out_reason) const;
 
     /// Can we assign a merge with these two parts?
     /// (assuming that no merge was assigned after the predicate was constructed)
     /// If we can't and out_reason is not nullptr, set it to the reason why we can't merge.
     bool canMergeTwoParts(const MergeTreeData::DataPartPtr & left,
                           const MergeTreeData::DataPartPtr & right,
-                          String * out_reason = nullptr) const;
+                          String & out_reason) const;
 
     /// Can we assign a merge this part and some other part?
     /// For example a merge of a part and itself is needed for TTL.
     /// This predicate is checked for the first part of each range.
-    bool canMergeSinglePart(const MergeTreeData::DataPartPtr & part, String * out_reason) const;
+    bool canMergeSinglePart(const MergeTreeData::DataPartPtr & part, String & out_reason) const;
 
     CommittingBlocks getCommittingBlocks(zkutil::ZooKeeperPtr & zookeeper, const std::string & zookeeper_path, Poco::Logger * log_);
 
@@ -561,7 +561,7 @@ public:
 
     /// Returns true if part is needed for some REPLACE_RANGE entry.
     /// We should not drop part in this case, because replication queue may stuck without that part.
-    bool partParticipatesInReplaceRange(const MergeTreeData::DataPartPtr & part, String * out_reason) const;
+    bool partParticipatesInReplaceRange(const MergeTreeData::DataPartPtr & part, String & out_reason) const;
 
     /// Return nonempty optional of desired mutation version and alter version.
     /// If we have no alter (modify/drop) mutations in mutations queue, than we return biggest possible

src/Storages/StorageDistributed.cpp

@@ -60,7 +60,6 @@
 #include <Interpreters/Cluster.h>
 #include <Interpreters/DatabaseAndTableWithAlias.h>
 #include <Interpreters/ExpressionAnalyzer.h>
-#include <Interpreters/InterpreterDescribeQuery.h>
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Interpreters/InterpreterSelectQueryAnalyzer.h>
 #include <Interpreters/InterpreterInsertQuery.h>
@@ -75,6 +74,7 @@
 #include <Interpreters/getTableExpressions.h>
 #include <Interpreters/RequiredSourceColumnsVisitor.h>
 #include <Interpreters/getCustomKeyFilterForParallelReplicas.h>
+#include <Interpreters/getHeaderForProcessingStage.h>
 
 #include <Functions/IFunction.h>
 #include <Functions/FunctionFactory.h>
@@ -434,7 +434,7 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(
         {
             /// Always calculate optimized cluster here, to avoid conditions during read()
             /// (Anyway it will be calculated in the read())
-            ClusterPtr optimized_cluster = getOptimizedCluster(local_context, storage_snapshot, query_info.query);
+            ClusterPtr optimized_cluster = getOptimizedCluster(local_context, storage_snapshot, query_info);
             if (optimized_cluster)
             {
                 LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): {}",
@@ -1297,7 +1297,7 @@ ClusterPtr StorageDistributed::getCluster() const
 }
 
 ClusterPtr StorageDistributed::getOptimizedCluster(
-    ContextPtr local_context, const StorageSnapshotPtr & storage_snapshot, const ASTPtr & query_ptr) const
+    ContextPtr local_context, const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info) const
 {
     ClusterPtr cluster = getCluster();
     const Settings & settings = local_context->getSettingsRef();
@@ -1306,7 +1306,7 @@ ClusterPtr StorageDistributed::getOptimizedCluster(
 
     if (has_sharding_key && sharding_key_is_usable)
     {
-        ClusterPtr optimized = skipUnusedShards(cluster, query_ptr, storage_snapshot, local_context);
+        ClusterPtr optimized = skipUnusedShards(cluster, query_info, storage_snapshot, local_context);
         if (optimized)
             return optimized;
     }
@@ -1355,25 +1355,34 @@ IColumn::Selector StorageDistributed::createSelector(const ClusterPtr cluster, c
 /// using constraints from "PREWHERE" and "WHERE" conditions, otherwise returns `nullptr`
 ClusterPtr StorageDistributed::skipUnusedShards(
     ClusterPtr cluster,
-    const ASTPtr & query_ptr,
+    const SelectQueryInfo & query_info,
     const StorageSnapshotPtr & storage_snapshot,
     ContextPtr local_context) const
 {
-    const auto & select = query_ptr->as<ASTSelectQuery &>();
-
+    const auto & select = query_info.query->as<ASTSelectQuery &>();
     if (!select.prewhere() && !select.where())
-    {
         return nullptr;
-    }
+
+    /// FIXME: support analyzer
+    if (!query_info.syntax_analyzer_result)
+        return nullptr;
 
     ASTPtr condition_ast;
-    if (select.prewhere() && select.where())
+    /// Remove JOIN from the query since it may contain a condition for other tables.
+    /// But only the conditions for the left table should be analyzed for shard skipping.
     {
-        condition_ast = makeASTFunction("and", select.prewhere()->clone(), select.where()->clone());
-    }
-    else
-    {
-        condition_ast = select.prewhere() ? select.prewhere()->clone() : select.where()->clone();
+        ASTPtr select_without_join_ptr = select.clone();
+        ASTSelectQuery select_without_join = select_without_join_ptr->as<ASTSelectQuery &>();
+        TreeRewriterResult analyzer_result_without_join = *query_info.syntax_analyzer_result;
+
+        removeJoin(select_without_join, analyzer_result_without_join, local_context);
+        if (!select_without_join.prewhere() && !select_without_join.where())
+            return nullptr;
+
+        if (select_without_join.prewhere() && select_without_join.where())
+            condition_ast = makeASTFunction("and", select_without_join.prewhere()->clone(), select_without_join.where()->clone());
+        else
+            condition_ast = select_without_join.prewhere() ? select_without_join.prewhere()->clone() : select_without_join.where()->clone();
     }
 
     replaceConstantExpressions(condition_ast, local_context, storage_snapshot->metadata->getColumns().getAll(), shared_from_this(), storage_snapshot);
@@ -1396,11 +1405,9 @@ ClusterPtr StorageDistributed::skipUnusedShards(
         return nullptr;
     }
 
-    // Can't get definite answer if we can skip any shards
+    // Can't get a definite answer if we can skip any shards
     if (!blocks)
-    {
         return nullptr;
-    }
 
     std::set<int> shards;
 

src/Storages/StorageDistributed.h

@@ -182,10 +182,10 @@ private:
     /// Apply the following settings:
     /// - optimize_skip_unused_shards
     /// - force_optimize_skip_unused_shards
-    ClusterPtr getOptimizedCluster(ContextPtr, const StorageSnapshotPtr & storage_snapshot, const ASTPtr & query_ptr) const;
+    ClusterPtr getOptimizedCluster(ContextPtr, const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info) const;
 
     ClusterPtr skipUnusedShards(
-        ClusterPtr cluster, const ASTPtr & query_ptr, const StorageSnapshotPtr & storage_snapshot, ContextPtr context) const;
+        ClusterPtr cluster, const SelectQueryInfo & query_info, const StorageSnapshotPtr & storage_snapshot, ContextPtr context) const;
 
     /// This method returns optimal query processing stage.
     ///

src/Storages/StorageMergeTree.cpp

@@ -45,6 +45,7 @@
 #include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
 #include <fmt/core.h>
 
+
 namespace DB
 {
 
@@ -857,7 +858,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
     bool aggressive,
     const String & partition_id,
     bool final,
-    String * out_disable_reason,
+    String & out_disable_reason,
     TableLockHolder & /* table_lock_holder */,
     std::unique_lock<std::mutex> & lock,
     const MergeTreeTransactionPtr & txn,
@@ -875,7 +876,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
     CurrentlyMergingPartsTaggerPtr merging_tagger;
     MergeList::EntryPtr merge_entry;
 
-    auto can_merge = [this, &lock](const DataPartPtr & left, const DataPartPtr & right, const MergeTreeTransaction * tx, String * disable_reason) -> bool
+    auto can_merge = [this, &lock](const DataPartPtr & left, const DataPartPtr & right, const MergeTreeTransaction * tx, String & disable_reason) -> bool
     {
         if (tx)
         {
@@ -884,8 +885,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
             if ((left && !left->version.isVisible(tx->getSnapshot(), Tx::EmptyTID))
                     || (right && !right->version.isVisible(tx->getSnapshot(), Tx::EmptyTID)))
             {
-                if (disable_reason)
-                    *disable_reason = "Some part is not visible in transaction";
+                disable_reason = "Some part is not visible in transaction";
                 return false;
             }
 
@@ -893,8 +893,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
             if ((left && left->version.isRemovalTIDLocked())
                     || (right && right->version.isRemovalTIDLocked()))
             {
-                if (disable_reason)
-                    *disable_reason = "Some part is locked for removal in another cuncurrent transaction";
+                disable_reason = "Some part is locked for removal in another cuncurrent transaction";
                 return false;
             }
         }
@@ -905,8 +904,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
         {
             if (currently_merging_mutating_parts.contains(right))
             {
-                if (disable_reason)
-                    *disable_reason = "Some part currently in a merging or mutating process";
+                disable_reason = "Some part currently in a merging or mutating process";
                 return false;
             }
             else
@@ -915,30 +913,26 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
 
         if (currently_merging_mutating_parts.contains(left) || currently_merging_mutating_parts.contains(right))
         {
-            if (disable_reason)
-                *disable_reason = "Some part currently in a merging or mutating process";
+            disable_reason = "Some part currently in a merging or mutating process";
             return false;
         }
 
         if (getCurrentMutationVersion(left, lock) != getCurrentMutationVersion(right, lock))
         {
-            if (disable_reason)
-                *disable_reason = "Some parts have differ mmutatuon version";
+            disable_reason = "Some parts have differ mmutatuon version";
             return false;
         }
 
         if (!partsContainSameProjections(left, right))
         {
-            if (disable_reason)
-                *disable_reason = "Some parts contains differ projections";
+            disable_reason = "Some parts contains differ projections";
             return false;
         }
 
         auto max_possible_level = getMaxLevelInBetween(left, right);
         if (max_possible_level > std::max(left->info.level, right->info.level))
         {
-            if (disable_reason)
-                *disable_reason = fmt::format("There is an outdated part in a gap between two active parts ({}, {}) with merge level {} higher than these active parts have", left->name, right->name, max_possible_level);
+            disable_reason = fmt::format("There is an outdated part in a gap between two active parts ({}, {}) with merge level {} higher than these active parts have", left->name, right->name, max_possible_level);
             return false;
         }
 
@@ -947,14 +941,13 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
 
     SelectPartsDecision select_decision = SelectPartsDecision::CANNOT_SELECT;
 
-    auto is_background_memory_usage_ok = [](String * disable_reason) -> bool
+    auto is_background_memory_usage_ok = [](String & disable_reason) -> bool
     {
         if (canEnqueueBackgroundTask())
             return true;
-        if (disable_reason)
-            *disable_reason = fmt::format("Current background tasks memory usage ({}) is more than the limit ({})",
-                formatReadableSizeWithBinarySuffix(background_memory_tracker.get()),
-                formatReadableSizeWithBinarySuffix(background_memory_tracker.getSoftLimit()));
+        disable_reason = fmt::format("Current background tasks memory usage ({}) is more than the limit ({})",
+            formatReadableSizeWithBinarySuffix(background_memory_tracker.get()),
+            formatReadableSizeWithBinarySuffix(background_memory_tracker.getSoftLimit()));
         return false;
     };
 
@@ -979,8 +972,8 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
                     txn,
                     out_disable_reason);
             }
-            else if (out_disable_reason)
-                *out_disable_reason = "Current value of max_source_parts_size is zero";
+            else
+                out_disable_reason = "Current value of max_source_parts_size is zero";
         }
     }
     else
@@ -1014,15 +1007,14 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
             /// If final - we will wait for currently processing merges to finish and continue.
             if (final
                 && select_decision != SelectPartsDecision::SELECTED
-                && !currently_merging_mutating_parts.empty()
-                && out_disable_reason)
+                && !currently_merging_mutating_parts.empty())
             {
                 LOG_DEBUG(log, "Waiting for currently running merges ({} parts are merging right now) to perform OPTIMIZE FINAL",
                     currently_merging_mutating_parts.size());
 
                 if (std::cv_status::timeout == currently_processing_in_background_condition.wait_for(lock, timeout))
                 {
-                    *out_disable_reason = fmt::format("Timeout ({} ms) while waiting for already running merges before running OPTIMIZE with FINAL", timeout_ms);
+                    out_disable_reason = fmt::format("Timeout ({} ms) while waiting for already running merges before running OPTIMIZE with FINAL", timeout_ms);
                     break;
                 }
             }
@@ -1038,14 +1030,9 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
 
     if (select_decision != SelectPartsDecision::SELECTED)
     {
-        if (out_disable_reason)
-        {
-            if (!out_disable_reason->empty())
-            {
-                *out_disable_reason += ". ";
-            }
-            *out_disable_reason += "Cannot select parts for optimization";
-        }
+        if (!out_disable_reason.empty())
+            out_disable_reason += ". ";
+        out_disable_reason += "Cannot select parts for optimization";
 
         return {};
     }
@@ -1066,7 +1053,7 @@ bool StorageMergeTree::merge(
     const Names & deduplicate_by_columns,
     bool cleanup,
     const MergeTreeTransactionPtr & txn,
-    String * out_disable_reason,
+    String & out_disable_reason,
     bool optimize_skip_merged_partitions)
 {
     auto table_lock_holder = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations);
@@ -1121,7 +1108,7 @@ bool StorageMergeTree::partIsAssignedToBackgroundOperation(const DataPartPtr & p
 }
 
 MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate(
-    const StorageMetadataPtr & metadata_snapshot, String * /* disable_reason */, TableLockHolder & /* table_lock_holder */,
+    const StorageMetadataPtr & metadata_snapshot, String & /* disable_reason */, TableLockHolder & /* table_lock_holder */,
     std::unique_lock<std::mutex> & /*currently_processing_in_background_mutex_lock*/)
 {
     if (current_mutations_by_version.empty())
@@ -1322,10 +1309,11 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign
         if (merger_mutator.merges_blocker.isCancelled())
             return false;
 
-        merge_entry = selectPartsToMerge(metadata_snapshot, false, {}, false, nullptr, shared_lock, lock, txn);
+        String out_reason;
+        merge_entry = selectPartsToMerge(metadata_snapshot, false, {}, false, out_reason, shared_lock, lock, txn);
 
         if (!merge_entry && !current_mutations_by_version.empty())
-            mutate_entry = selectPartsToMutate(metadata_snapshot, nullptr, shared_lock, lock);
+            mutate_entry = selectPartsToMutate(metadata_snapshot, out_reason, shared_lock, lock);
 
         has_mutations = !current_mutations_by_version.empty();
     }
@@ -1528,7 +1516,7 @@ bool StorageMergeTree::optimize(
                     deduplicate_by_columns,
                     cleanup,
                     txn,
-                    &disable_reason,
+                    disable_reason,
                     local_context->getSettingsRef().optimize_skip_merged_partitions))
             {
                 constexpr auto message = "Cannot OPTIMIZE table: {}";
@@ -1556,7 +1544,7 @@ bool StorageMergeTree::optimize(
                 deduplicate_by_columns,
                 cleanup,
                 txn,
-                &disable_reason,
+                disable_reason,
                 local_context->getSettingsRef().optimize_skip_merged_partitions))
         {
             constexpr auto message = "Cannot OPTIMIZE table: {}";

src/Storages/StorageMergeTree.h

@@ -176,7 +176,7 @@ private:
             const Names & deduplicate_by_columns,
             bool cleanup,
             const MergeTreeTransactionPtr & txn,
-            String * out_disable_reason = nullptr,
+            String & out_disable_reason,
             bool optimize_skip_merged_partitions = false);
 
     void renameAndCommitEmptyParts(MutableDataPartsVector & new_parts, Transaction & transaction);
@@ -203,7 +203,7 @@ private:
         bool aggressive,
         const String & partition_id,
         bool final,
-        String * disable_reason,
+        String & disable_reason,
         TableLockHolder & table_lock_holder,
         std::unique_lock<std::mutex> & lock,
         const MergeTreeTransactionPtr & txn,
@@ -212,7 +212,7 @@ private:
 
 
     MergeMutateSelectedEntryPtr selectPartsToMutate(
-        const StorageMetadataPtr & metadata_snapshot, String * disable_reason,
+        const StorageMetadataPtr & metadata_snapshot, String & disable_reason,
         TableLockHolder & table_lock_holder, std::unique_lock<std::mutex> & currently_processing_in_background_mutex_lock);
 
     /// For current mutations queue, returns maximum version of mutation for a part,

src/Storages/StorageReplicatedMergeTree.cpp

@@ -3469,9 +3469,10 @@ void StorageReplicatedMergeTree::mergeSelectingTask()
                 merge_pred.emplace(queue.getMergePredicate(zookeeper, partitions_to_merge_in));
         }
 
+        String out_reason;
         if (can_assign_merge &&
             merger_mutator.selectPartsToMerge(future_merged_part, false, max_source_parts_size_for_merge, *merge_pred,
-                                              merge_with_ttl_allowed, NO_TRANSACTION_PTR, nullptr, &partitions_to_merge_in) == SelectPartsDecision::SELECTED)
+                merge_with_ttl_allowed, NO_TRANSACTION_PTR, out_reason, &partitions_to_merge_in) == SelectPartsDecision::SELECTED)
         {
             create_result = createLogEntryToMergeParts(
                 zookeeper,
@@ -5429,13 +5430,13 @@ bool StorageReplicatedMergeTree::optimize(
             {
                 select_decision = merger_mutator.selectPartsToMerge(
                     future_merged_part, /* aggressive */ true, storage_settings_ptr->max_bytes_to_merge_at_max_space_in_pool,
-                    can_merge, /* merge_with_ttl_allowed */ false, NO_TRANSACTION_PTR, &disable_reason);
+                    can_merge, /* merge_with_ttl_allowed */ false, NO_TRANSACTION_PTR, disable_reason);
             }
             else
             {
                 select_decision = merger_mutator.selectAllPartsToMergeWithinPartition(
                     future_merged_part, can_merge, partition_id, final, metadata_snapshot, NO_TRANSACTION_PTR,
-                    &disable_reason, query_context->getSettingsRef().optimize_skip_merged_partitions);
+                    disable_reason, query_context->getSettingsRef().optimize_skip_merged_partitions);
             }
 
             /// If there is nothing to merge then we treat this merge as successful (needed for optimize final optimization)
@@ -7974,7 +7975,7 @@ void StorageReplicatedMergeTree::movePartitionToShard(
 
         /// canMergeSinglePart is overlapping with dropPart, let's try to use the same code.
         String out_reason;
-        if (!merge_pred.canMergeSinglePart(part, &out_reason))
+        if (!merge_pred.canMergeSinglePart(part, out_reason))
             throw Exception(ErrorCodes::PART_IS_TEMPORARILY_LOCKED, "Part is busy, reason: {}", out_reason);
     }
 
@@ -8222,14 +8223,14 @@ bool StorageReplicatedMergeTree::dropPartImpl(
         /// There isn't a lot we can do otherwise. Can't cancel merges because it is possible that a replica already
         /// finished the merge.
         String out_reason;
-        if (!merge_pred.canMergeSinglePart(part, &out_reason))
+        if (!merge_pred.canMergeSinglePart(part, out_reason))
         {
             if (throw_if_noop)
                 throw Exception::createDeprecated(out_reason, ErrorCodes::PART_IS_TEMPORARILY_LOCKED);
             return false;
         }
 
-        if (merge_pred.partParticipatesInReplaceRange(part, &out_reason))
+        if (merge_pred.partParticipatesInReplaceRange(part, out_reason))
         {
             if (throw_if_noop)
                 throw Exception::createDeprecated(out_reason, ErrorCodes::PART_IS_TEMPORARILY_LOCKED);

src/Storages/getStructureOfRemoteTable.cpp

@@ -2,7 +2,6 @@
 #include <Interpreters/Cluster.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/ClusterProxy/executeQuery.h>
-#include <Interpreters/InterpreterDescribeQuery.h>
 #include <QueryPipeline/RemoteQueryExecutor.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeString.h>

tests/analyzer_tech_debt.txt

@@ -130,4 +130,6 @@
 02581_share_big_sets_between_mutation_tasks_long
 02581_share_big_sets_between_multiple_mutations_tasks_long
 00992_system_parts_race_condition_zookeeper_long
+02790_optimize_skip_unused_shards_join
+01940_custom_tld_sharding_key
 02815_range_dict_no_direct_join

tests/ci/docker_images_check.py

@@ -8,6 +8,7 @@ import shutil
 import subprocess
 import time
 import sys
+from glob import glob
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
@@ -31,6 +32,17 @@ TEMP_PATH = os.path.join(RUNNER_TEMP, "docker_images_check")
 ImagesDict = Dict[str, dict]
 
 
+# workaround for mypy issue [1]:
+#
+#    "Argument 1 to "map" has incompatible type overloaded function" [1]
+#
+#  [1]: https://github.com/python/mypy/issues/9864
+#
+# NOTE: simply lambda will do the trick as well, but pylint will not like it
+def realpath(*args, **kwargs):
+    return os.path.realpath(*args, **kwargs)
+
+
 class DockerImage:
     def __init__(
         self,
@@ -111,8 +123,23 @@ def get_changed_docker_images(
     changed_images = []
 
     for dockerfile_dir, image_description in images_dict.items():
+        source_dir = GITHUB_WORKSPACE.rstrip("/") + "/"
+        dockerfile_files = glob(f"{source_dir}/{dockerfile_dir}/**", recursive=True)
+        # resolve symlinks
+        dockerfile_files = list(map(realpath, dockerfile_files))
+        # trim prefix to get relative path again, to match with files_changed
+        dockerfile_files = list(map(lambda x: x[len(source_dir) :], dockerfile_files))
+        logging.info(
+            "Docker %s (source_dir=%s) build context for PR %s @ %s: %s",
+            dockerfile_dir,
+            source_dir,
+            pr_info.number,
+            pr_info.sha,
+            str(dockerfile_files),
+        )
+
         for f in files_changed:
-            if f.startswith(dockerfile_dir):
+            if f in dockerfile_files:
                 name = image_description["name"]
                 only_amd64 = image_description.get("only_amd64", False)
                 logging.info(
@@ -245,6 +272,8 @@ def build_and_push_one_image(
         cache_from = f"{cache_from} --cache-from type=registry,ref={image.repo}:{tag}"
 
     cmd = (
+        # tar is requried to follow symlinks, since docker-build cannot do this
+        f"tar -v --exclude-vcs-ignores --show-transformed-names --transform 's#{image.full_path.lstrip('/')}#./#' --dereference --create {image.full_path} | "
         "docker buildx build --builder default "
         f"--label build-url={GITHUB_RUN_URL} "
         f"{from_tag_arg}"
@@ -254,7 +283,7 @@ def build_and_push_one_image(
         f"{cache_from} "
         f"--cache-to type=inline,mode=max "
         f"{push_arg}"
-        f"--progress plain {image.full_path}"
+        f"--progress plain -"
     )
     logging.info("Docker command to run: %s", cmd)
     with TeePopen(cmd, build_log) as proc:

tests/ci/docker_test.py

@@ -126,12 +126,13 @@ class TestDockerImageCheck(unittest.TestCase):
         mock_popen.assert_called_once()
         mock_machine.assert_not_called()
         self.assertIn(
+            "tar -v --exclude-vcs-ignores --show-transformed-names --transform 's#path#./#' --dereference --create path | "
             f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} "
             "--build-arg FROM_TAG=version "
             f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
             "--tag name:version --cache-from type=registry,ref=name:version "
             "--cache-from type=registry,ref=name:latest "
-            "--cache-to type=inline,mode=max --push --progress plain path",
+            "--cache-to type=inline,mode=max --push --progress plain -",
             mock_popen.call_args.args,
         )
         self.assertTrue(result)
@@ -143,12 +144,13 @@ class TestDockerImageCheck(unittest.TestCase):
         mock_popen.assert_called_once()
         mock_machine.assert_not_called()
         self.assertIn(
+            "tar -v --exclude-vcs-ignores --show-transformed-names --transform 's#path#./#' --dereference --create path | "
             f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} "
             "--build-arg FROM_TAG=version2 "
             f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
             "--tag name:version2 --cache-from type=registry,ref=name:version2 "
             "--cache-from type=registry,ref=name:latest "
-            "--cache-to type=inline,mode=max --progress plain path",
+            "--cache-to type=inline,mode=max --progress plain -",
             mock_popen.call_args.args,
         )
         self.assertTrue(result)
@@ -160,11 +162,12 @@ class TestDockerImageCheck(unittest.TestCase):
         mock_popen.assert_called_once()
         mock_machine.assert_not_called()
         self.assertIn(
+            "tar -v --exclude-vcs-ignores --show-transformed-names --transform 's#path#./#' --dereference --create path | "
             f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} "
             f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
             "--tag name:version2 --cache-from type=registry,ref=name:version2 "
             "--cache-from type=registry,ref=name:latest "
-            "--cache-to type=inline,mode=max --progress plain path",
+            "--cache-to type=inline,mode=max --progress plain -",
             mock_popen.call_args.args,
         )
         self.assertFalse(result)
@@ -178,13 +181,14 @@ class TestDockerImageCheck(unittest.TestCase):
         mock_popen.assert_called_once()
         mock_machine.assert_not_called()
         self.assertIn(
+            "tar -v --exclude-vcs-ignores --show-transformed-names --transform 's#path#./#' --dereference --create path | "
             f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} "
             f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} "
             "--tag name:version2 --cache-from type=registry,ref=name:version2 "
             "--cache-from type=registry,ref=name:latest "
             "--cache-from type=registry,ref=name:cached-version "
             "--cache-from type=registry,ref=name:another-cached "
-            "--cache-to type=inline,mode=max --progress plain path",
+            "--cache-to type=inline,mode=max --progress plain -",
             mock_popen.call_args.args,
         )
         self.assertFalse(result)

tests/integration/ci-runner.py

@@ -190,7 +190,7 @@ def clear_ip_tables_and_restart_daemons():
     try:
         logging.info("Killing all alive docker containers")
         subprocess.check_output(
-            "timeout -s 9 10m docker ps --quiet | xargs --no-run-if-empty docker kill",
+            "timeout --signal=KILL 10m docker ps --quiet | xargs --no-run-if-empty docker kill",
             shell=True,
         )
     except subprocess.CalledProcessError as err:
@@ -199,7 +199,7 @@ def clear_ip_tables_and_restart_daemons():
     try:
         logging.info("Removing all docker containers")
         subprocess.check_output(
-            "timeout -s 9 10m docker ps --all --quiet | xargs --no-run-if-empty docker rm --force",
+            "timeout --signal=KILL 10m docker ps --all --quiet | xargs --no-run-if-empty docker rm --force",
             shell=True,
         )
     except subprocess.CalledProcessError as err:
@@ -321,7 +321,7 @@ class ClickhouseIntegrationTestsRunner:
 
         cmd = (
             "cd {repo_path}/tests/integration && "
-            "timeout -s 9 1h ./runner {runner_opts} {image_cmd} --pre-pull --command '{command}' ".format(
+            "timeout --signal=KILL 1h ./runner {runner_opts} {image_cmd} --pre-pull --command '{command}' ".format(
                 repo_path=repo_path,
                 runner_opts=self._get_runner_opts(),
                 image_cmd=image_cmd,
@@ -433,9 +433,9 @@ class ClickhouseIntegrationTestsRunner:
         out_file_full = os.path.join(self.result_path, "runner_get_all_tests.log")
         cmd = (
             "cd {repo_path}/tests/integration && "
-            "timeout -s 9 1h ./runner {runner_opts} {image_cmd} -- --setup-plan "
-            "| tee {out_file_full} | grep '::' | sed 's/ (fixtures used:.*//g' | sed 's/^ *//g' | sed 's/ *$//g' "
-            "| grep -v 'SKIPPED' | sort -u  > {out_file}".format(
+            "timeout --signal=KILL 1h ./runner {runner_opts} {image_cmd} -- --setup-plan "
+            "| tee '{out_file_full}' | grep -F '::' | sed -r 's/ \(fixtures used:.*//g; s/^ *//g; s/ *$//g' "
+            "| grep -v -F 'SKIPPED' | sort --unique > {out_file}".format(
                 repo_path=repo_path,
                 runner_opts=self._get_runner_opts(),
                 image_cmd=image_cmd,
@@ -677,7 +677,7 @@ class ClickhouseIntegrationTestsRunner:
             # -E -- (E)rror
             # -p -- (p)assed
             # -s -- (s)kipped
-            cmd = "cd {}/tests/integration && timeout -s 9 1h ./runner {} {} -t {} {} -- -rfEps --run-id={} --color=no --durations=0 {} | tee {}".format(
+            cmd = "cd {}/tests/integration && timeout --signal=KILL 1h ./runner {} {} -t {} {} -- -rfEps --run-id={} --color=no --durations=0 {} | tee {}".format(
                 repo_path,
                 self._get_runner_opts(),
                 image_cmd,

tests/integration/conftest.py

@@ -12,6 +12,22 @@ from helpers.network import _NetworkManager
 logging.raiseExceptions = False
 
 
+@pytest.fixture(autouse=True, scope="session")
+def tune_local_port_range():
+    # Lots of services uses non privileged ports:
+    # - hdfs -- 50020/50070/...
+    # - minio
+    # - mysql
+    # - psql
+    #
+    # So instead of tuning all these thirdparty services, let's simply
+    # prohibit using such ports for outgoing connections, this should fix
+    # possible "Address already in use" errors.
+    #
+    # NOTE: 5K is not enough, and sometimes leads to EADDRNOTAVAIL error.
+    run_and_check(["sysctl net.ipv4.ip_local_port_range='55000 65535'"], shell=True)
+
+
 @pytest.fixture(autouse=True, scope="session")
 def cleanup_environment():
     try:

tests/integration/helpers/cluster.py

@@ -3199,6 +3199,7 @@ class ClickHouseInstance:
     ):
         self.name = name
         self.base_cmd = cluster.base_cmd
+        self.base_dir = base_path
         self.docker_id = cluster.get_instance_docker_id(self.name)
         self.cluster = cluster
         self.hostname = hostname if hostname is not None else self.name
@@ -4193,6 +4194,14 @@ class ClickHouseInstance:
             ["bash", "-c", f"sed -i 's/{replace}/{replacement}/g' {path_to_config}"]
         )
 
+    def put_users_config(self, config_path):
+        """Put new config (useful if you cannot put it at the start)"""
+
+        instance_config_dir = p.abspath(p.join(self.path, "configs"))
+        users_d_dir = p.abspath(p.join(instance_config_dir, "users.d"))
+        config_path = p.join(self.base_dir, config_path)
+        shutil.copy(config_path, users_d_dir)
+
     def create_dir(self):
         """Create the instance directory and all the needed files there."""
 

tests/integration/test_azure_blob_storage_zero_copy_replication/configs/config.d/storage_conf.xml

@@ -45,5 +45,6 @@
 
     <merge_tree>
         <allow_remote_fs_zero_copy_replication>true</allow_remote_fs_zero_copy_replication>
+        <ratio_of_defaults_for_sparse_serialization>1.0</ratio_of_defaults_for_sparse_serialization>
     </merge_tree>
 </clickhouse>

tests/integration/test_backward_compatibility/configs/no_allow_vertical_merges_from_compact_to_wide_parts.xml

@@ -0,0 +1,5 @@
+<clickhouse>
+    <merge_tree>
+        <allow_vertical_merges_from_compact_to_wide_parts>0</allow_vertical_merges_from_compact_to_wide_parts>
+    </merge_tree>
+</clickhouse>

tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py

@@ -15,7 +15,10 @@ node_old = cluster.add_instance(
 )
 node_new = cluster.add_instance(
     "node2",
-    main_configs=["configs/no_compress_marks.xml"],
+    main_configs=[
+        "configs/no_compress_marks.xml",
+        "configs/no_allow_vertical_merges_from_compact_to_wide_parts.xml",
+    ],
     with_zookeeper=True,
     stay_alive=True,
     allow_analyzer=False,

tests/integration/test_materialized_mysql_database/materialized_with_ddl.py

@@ -980,6 +980,89 @@ def query_event_with_empty_transaction(clickhouse_node, mysql_node, service_name
     mysql_node.query("DROP DATABASE test_database_event")
 
 
+def text_blob_with_charset_test(clickhouse_node, mysql_node, service_name):
+    db = "text_blob_with_charset_test"
+    mysql_node.query(f"DROP DATABASE IF EXISTS {db}")
+    clickhouse_node.query(f"DROP DATABASE IF EXISTS {db}")
+    mysql_node.query(f"CREATE DATABASE {db} DEFAULT CHARACTER SET 'utf8'")
+
+    mysql_node.query(
+        f"CREATE TABLE {db}.test_table_1 (a INT NOT NULL PRIMARY KEY, b text CHARACTER SET gbk, c tinytext CHARSET big5, d longtext, e varchar(256), f char(4)) ENGINE = InnoDB DEFAULT CHARSET=gbk"
+    )
+    mysql_node.query(
+        f"CREATE TABLE {db}.test_table_2 (a INT NOT NULL PRIMARY KEY, b blob, c longblob) ENGINE = InnoDB DEFAULT CHARSET=gbk"
+    )
+    mysql_node.query(
+        f"CREATE TABLE {db}.test_table_3 (a INT NOT NULL PRIMARY KEY, b text CHARACTER SET gbk, c tinytext CHARSET gbk, d tinytext CHARSET big5, e varchar(256), f char(4)) ENGINE = InnoDB"
+    )
+
+    mysql_node.query(
+        f"INSERT INTO {db}.test_table_1 VALUES (1, '你好', '世界', '哈罗', '您Hi您', '您Hi您')"
+    )
+    mysql_node.query(
+        f"INSERT INTO {db}.test_table_2 VALUES (1, '你好', 0xFAAA00000000000DDCC)"
+    )
+    mysql_node.query(
+        f"INSERT INTO {db}.test_table_3 VALUES (1, '你好', '世界', 'hello', '您Hi您', '您Hi您')"
+    )
+
+    clickhouse_node.query(
+        f"CREATE DATABASE {db} ENGINE = MaterializedMySQL('{service_name}:3306', '{db}', 'root', 'clickhouse')"
+    )
+    assert db in clickhouse_node.query("SHOW DATABASES")
+
+    # from full replication
+    check_query(
+        clickhouse_node,
+        f"SHOW TABLES FROM {db} FORMAT TSV",
+        "test_table_1\ntest_table_2\ntest_table_3\n",
+    )
+    check_query(
+        clickhouse_node,
+        f"SELECT b, c, d, e, f FROM {db}.test_table_1 WHERE a = 1 FORMAT TSV",
+        "你好\t世界\t哈罗\t您Hi您\t您Hi您\n",
+    )
+    check_query(
+        clickhouse_node,
+        f"SELECT hex(b), hex(c) FROM {db}.test_table_2 WHERE a = 1 FORMAT TSV",
+        "E4BDA0E5A5BD\t0FAAA00000000000DDCC\n",
+    )
+    check_query(
+        clickhouse_node,
+        f"SELECT b, c, d, e, f FROM {db}.test_table_3 WHERE a = 1 FORMAT TSV",
+        "你好\t世界\thello\t您Hi您\t您Hi您\n",
+    )
+
+    # from increment replication
+    mysql_node.query(
+        f"INSERT INTO {db}.test_table_1 VALUES (2, '你好', '世界', '哈罗', '您Hi您', '您Hi您')"
+    )
+    mysql_node.query(
+        f"INSERT INTO {db}.test_table_2 VALUES (2, '你好', 0xFAAA00000000000DDCC)"
+    )
+    mysql_node.query(
+        f"INSERT INTO {db}.test_table_3 VALUES (2, '你好', '世界', 'hello', '您Hi您', '您Hi您')"
+    )
+
+    check_query(
+        clickhouse_node,
+        f"SELECT b, c, d, e, f FROM {db}.test_table_1 WHERE a = 2 FORMAT TSV",
+        "你好\t世界\t哈罗\t您Hi您\t您Hi您\n",
+    )
+    check_query(
+        clickhouse_node,
+        f"SELECT hex(b), hex(c) FROM {db}.test_table_2 WHERE a = 2 FORMAT TSV",
+        "E4BDA0E5A5BD\t0FAAA00000000000DDCC\n",
+    )
+    check_query(
+        clickhouse_node,
+        f"SELECT b, c, d, e, f FROM {db}.test_table_3 WHERE a = 2 FORMAT TSV",
+        "你好\t世界\thello\t您Hi您\t您Hi您\n",
+    )
+    clickhouse_node.query(f"DROP DATABASE {db}")
+    mysql_node.query(f"DROP DATABASE {db}")
+
+
 def select_without_columns(clickhouse_node, mysql_node, service_name):
     mysql_node.query("DROP DATABASE IF EXISTS db")
     clickhouse_node.query("DROP DATABASE IF EXISTS db")

tests/integration/test_materialized_mysql_database/test.py

@@ -262,6 +262,12 @@ def test_materialized_database_ddl_with_empty_transaction_8_0(
     )
 
 
+def test_text_blob_charset(started_cluster, started_mysql_8_0, clickhouse_node):
+    materialized_with_ddl.text_blob_with_charset_test(
+        clickhouse_node, started_mysql_8_0, "mysql80"
+    )
+
+
 def test_select_without_columns_5_7(
     started_cluster, started_mysql_5_7, clickhouse_node
 ):

tests/integration/test_memory_profiler_min_max_borders/__init__.py

@@ -1 +0,0 @@
-#!/usr/bin/env python3

tests/integration/test_memory_profiler_min_max_borders/configs/memory_profiler.xml

@@ -1,5 +0,0 @@
-<clickhouse>
-    <total_memory_tracker_sample_probability>1</total_memory_tracker_sample_probability>
-    <total_memory_profiler_sample_min_allocation_size>4096</total_memory_profiler_sample_min_allocation_size>
-    <total_memory_profiler_sample_max_allocation_size>8192</total_memory_profiler_sample_max_allocation_size>
-</clickhouse>

tests/integration/test_memory_profiler_min_max_borders/test.py

@@ -1,37 +0,0 @@
-from helpers.cluster import ClickHouseCluster
-import pytest
-
-cluster = ClickHouseCluster(__file__)
-node = cluster.add_instance(
-    "node",
-    main_configs=["configs/memory_profiler.xml"],
-    user_configs=["configs/max_untracked_memory.xml"],
-)
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    try:
-        cluster.start()
-        yield cluster
-
-    finally:
-        cluster.shutdown()
-
-
-def test_trace_boundaries_work(started_cluster):
-    node.query("select randomPrintableASCII(number) from numbers(1000) FORMAT Null")
-    node.query("SYSTEM FLUSH LOGS")
-
-    assert (
-        node.query(
-            "SELECT countDistinct(abs(size)) > 0 FROM system.trace_log where trace_type = 'MemorySample'"
-        )
-        == "1\n"
-    )
-    assert (
-        node.query(
-            "SELECT count() FROM system.trace_log where trace_type = 'MemorySample' and (abs(size) > 8192 or abs(size) < 4096)"
-        )
-        == "0\n"
-    )

tests/integration/test_merge_tree_hdfs/configs/config.d/storage_conf.xml

@@ -28,6 +28,7 @@
 
     <merge_tree>
         <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
+        <ratio_of_defaults_for_sparse_serialization>1.0</ratio_of_defaults_for_sparse_serialization>
     </merge_tree>
     <allow_remove_stale_moving_parts>true</allow_remove_stale_moving_parts>
 </clickhouse>