Merge remote-tracking branch 'origin/master' into fix-usage-of-plain-metadata

.github/workflows/master.yml

@@ -305,7 +305,7 @@ jobs:
       runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
   MarkReleaseReady:
-    if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
+    if: ${{ !failure() && !cancelled() }}
     needs:
       - BuilderBinDarwin
       - BuilderBinDarwinAarch64
@@ -313,9 +313,25 @@ jobs:
       - BuilderDebAarch64
     runs-on: [self-hosted, style-checker]
     steps:
+      - name: Debug
+        run: |
+          echo need with different filters
+          cat << 'EOF'
+          ${{ toJSON(needs) }}
+          ${{ toJSON(needs.*.result) }}
+          no failures ${{ !contains(needs.*.result, 'failure') }}
+          no skips ${{ !contains(needs.*.result, 'skipped') }}
+          no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
+          EOF
+      - name: Not ready
+        # fail the job to be able restart it
+        if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
+        run: exit 1
       - name: Check out repository code
+        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
         uses: ClickHouse/checkout@v1
       - name: Mark Commit Release Ready
+        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 mark_release_ready.py

.github/workflows/release_branches.yml

@@ -206,7 +206,7 @@ jobs:
       runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
   MarkReleaseReady:
-    if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
+    if: ${{ !failure() && !cancelled() }}
     needs:
       - BuilderBinDarwin
       - BuilderBinDarwinAarch64
@@ -214,9 +214,25 @@ jobs:
       - BuilderDebAarch64
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
+      - name: Debug
+        run: |
+          echo need with different filters
+          cat << 'EOF'
+          ${{ toJSON(needs) }}
+          ${{ toJSON(needs.*.result) }}
+          no failures ${{ !contains(needs.*.result, 'failure') }}
+          no skips ${{ !contains(needs.*.result, 'skipped') }}
+          no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
+          EOF
+      - name: Not ready
+        # fail the job to be able restart it
+        if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
+        run: exit 1
       - name: Check out repository code
+        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
         uses: ClickHouse/checkout@v1
       - name: Mark Commit Release Ready
+        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 mark_release_ready.py

.github/workflows/reusable_build.yml

@@ -43,7 +43,8 @@ jobs:
     runs-on: [self-hosted, '${{inputs.runner_type}}']
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        # WIP: temporary try commit with limited perallelization of checkout
+        uses: ClickHouse/checkout@0be3f7b3098bae494d3ef5d29d2e0676fb606232
         with:
           clear-repository: true
           ref: ${{ fromJson(inputs.data).git_ref }}

.gitignore

@@ -165,7 +165,7 @@ tests/queries/0_stateless/*.expect.history
 tests/integration/**/_gen
 
 # rust
-/rust/**/target
+/rust/**/target*
 # It is autogenerated from *.in
 /rust/**/.cargo/config.toml
 /rust/**/vendor

CMakeLists.txt

@@ -319,7 +319,8 @@ if (COMPILER_CLANG)
     endif()
 endif ()
 
-set (COMPILER_FLAGS "${COMPILER_FLAGS}")
+# Disable floating-point expression contraction in order to get consistent floating point calculation results across platforms
+set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffp-contract=off")
 
 # Our built-in unwinder only supports DWARF version up to 4.
 set (DEBUG_INFO_FLAGS "-g")

README.md

@@ -31,15 +31,30 @@ curl https://clickhouse.com/ | sh
 * [Static Analysis (SonarCloud)](https://sonarcloud.io/project/issues?resolved=false&id=ClickHouse_ClickHouse) proposes C++ quality improvements.
 * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
 
+## Monthly Release & Community Call
+
+Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know.
+
+* [v24.3 Community Call](https://clickhouse.com/company/events/v24-3-community-release-call) - Mar 26
+* [v24.4 Community Call](https://clickhouse.com/company/events/v24-4-community-release-call) - Apr 30
+
 ## Upcoming Events
 
-Keep an eye out for upcoming meetups around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com.
+Keep an eye out for upcoming meetups and eventsaround the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
+
+* [ClickHouse Meetup in Bellevue](https://www.meetup.com/clickhouse-seattle-user-group/events/298650371/) - Mar 11
+* [ClickHouse Meetup at Ramp's Offices in NYC](https://www.meetup.com/clickhouse-new-york-user-group/events/298640542/) - Mar 19
+* [ClickHouse Melbourne Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/299479750/) - Mar 20
+* [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/298997115/) - Mar 21
+* [ClickHouse Meetup in Bengaluru](https://www.meetup.com/clickhouse-bangalore-user-group/events/299479850/) - Mar 23
+* [ClickHouse Meetup in Zurich](https://www.meetup.com/clickhouse-switzerland-meetup-group/events/299628922/) - Apr 16
+* [ClickHouse Meetup in Copenhagen](https://www.meetup.com/clickhouse-denmark-meetup-group/events/299629133/) - Apr 23
+* [ClickHouse Meetup in Dubai](https://www.meetup.com/clickhouse-dubai-meetup-group/events/299629189/) - May 28
+
 
 ## Recent Recordings
 * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
-* **Recording available**: [**v24.1 Release Webinar**](https://www.youtube.com/watch?v=pBF9g0wGAGs) All the features of 24.1, one convenient video! Watch it now!
+* **Recording available**: [**v24.2 Release Call**](https://www.youtube.com/watch?v=iN2y-TK8f3A) All the features of 24.2, one convenient video! Watch it now!
-* **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)
-
   
  ## Interested in joining ClickHouse and making it your full-time job? 
   

base/base/CMakeLists.txt

@@ -13,6 +13,7 @@ set (SRCS
     cgroupsv2.cpp
     coverage.cpp
     demangle.cpp
+    Decimal.cpp
     getAvailableMemoryAmount.cpp
     getFQDNOrHostName.cpp
     getMemoryAmount.cpp

base/base/Decimal.cpp

@@ -0,0 +1,87 @@
+#include <base/Decimal.h>
+#include <base/extended_types.h>
+
+namespace DB
+{
+
+/// Explicit template instantiations.
+
+#define FOR_EACH_UNDERLYING_DECIMAL_TYPE(M) \
+    M(Int32)  \
+    M(Int64)  \
+    M(Int128) \
+    M(Int256)
+
+#define FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(M, X) \
+    M(Int32, X) \
+    M(Int64, X) \
+    M(Int128, X) \
+    M(Int256, X)
+
+template <typename T> const Decimal<T> & Decimal<T>::operator += (const T & x) { value += x; return *this; }
+template <typename T> const Decimal<T> & Decimal<T>::operator -= (const T & x) { value -= x; return *this; }
+template <typename T> const Decimal<T> & Decimal<T>::operator *= (const T & x) { value *= x; return *this; }
+template <typename T> const Decimal<T> & Decimal<T>::operator /= (const T & x) { value /= x; return *this; }
+template <typename T> const Decimal<T> & Decimal<T>::operator %= (const T & x) { value %= x; return *this; }
+
+template <typename T> void NO_SANITIZE_UNDEFINED Decimal<T>::addOverflow(const T & x) { value += x; }
+
+/// Maybe this explicit instantiation affects performance since operators cannot be inlined.
+
+template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator += (const Decimal<U> & x) { value += static_cast<T>(x.value); return *this; }
+template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator -= (const Decimal<U> & x) { value -= static_cast<T>(x.value); return *this; }
+template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator *= (const Decimal<U> & x) { value *= static_cast<T>(x.value); return *this; }
+template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator /= (const Decimal<U> & x) { value /= static_cast<T>(x.value); return *this; }
+template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator %= (const Decimal<U> & x) { value %= static_cast<T>(x.value); return *this; }
+
+#define DISPATCH(TYPE_T, TYPE_U) \
+    template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator += (const Decimal<TYPE_U> & x); \
+    template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator -= (const Decimal<TYPE_U> & x); \
+    template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator *= (const Decimal<TYPE_U> & x); \
+    template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator /= (const Decimal<TYPE_U> & x); \
+    template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator %= (const Decimal<TYPE_U> & x);
+#define INVOKE(X) FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+#define DISPATCH(TYPE) template struct Decimal<TYPE>;
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
+
+template <typename T> bool operator< (const Decimal<T> & x, const Decimal<T> & y) { return x.value < y.value; }
+template <typename T> bool operator> (const Decimal<T> & x, const Decimal<T> & y) { return x.value > y.value; }
+template <typename T> bool operator<= (const Decimal<T> & x, const Decimal<T> & y) { return x.value <= y.value; }
+template <typename T> bool operator>= (const Decimal<T> & x, const Decimal<T> & y) { return x.value >= y.value; }
+template <typename T> bool operator== (const Decimal<T> & x, const Decimal<T> & y) { return x.value == y.value; }
+template <typename T> bool operator!= (const Decimal<T> & x, const Decimal<T> & y) { return x.value != y.value; }
+
+#define DISPATCH(TYPE) \
+template bool operator< (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template bool operator> (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template bool operator<= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template bool operator>= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template bool operator== (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template bool operator!= (const Decimal<TYPE> & x, const Decimal<TYPE> & y);
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
+
+
+template <typename T> Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y) { return x.value + y.value; }
+template <typename T> Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y) { return x.value - y.value; }
+template <typename T> Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y) { return x.value * y.value; }
+template <typename T> Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y) { return x.value / y.value; }
+template <typename T> Decimal<T> operator- (const Decimal<T> & x) { return -x.value; }
+
+#define DISPATCH(TYPE) \
+template Decimal<TYPE> operator+ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template Decimal<TYPE> operator- (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template Decimal<TYPE> operator* (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template Decimal<TYPE> operator/ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template Decimal<TYPE> operator- (const Decimal<TYPE> & x);
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
+
+#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS
+#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE
+}

base/base/Decimal.h

@@ -2,6 +2,7 @@
 
 #include <base/extended_types.h>
 #include <base/Decimal_fwd.h>
+#include <base/types.h>
 #include <base/defines.h>
 
 
@@ -10,6 +11,18 @@ namespace DB
 template <class> struct Decimal;
 class DateTime64;
 
+#define FOR_EACH_UNDERLYING_DECIMAL_TYPE(M) \
+    M(Int32) \
+    M(Int64) \
+    M(Int128) \
+    M(Int256)
+
+#define FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(M, X) \
+    M(Int32,  X) \
+    M(Int64,  X) \
+    M(Int128, X) \
+    M(Int256, X)
+
 using Decimal32 = Decimal<Int32>;
 using Decimal64 = Decimal<Int64>;
 using Decimal128 = Decimal<Int128>;
@@ -50,36 +63,73 @@ struct Decimal
             return static_cast<U>(value);
     }
 
-    const Decimal<T> & operator += (const T & x) { value += x; return *this; }
+    const Decimal<T> & operator += (const T & x);
-    const Decimal<T> & operator -= (const T & x) { value -= x; return *this; }
+    const Decimal<T> & operator -= (const T & x);
-    const Decimal<T> & operator *= (const T & x) { value *= x; return *this; }
+    const Decimal<T> & operator *= (const T & x);
-    const Decimal<T> & operator /= (const T & x) { value /= x; return *this; }
+    const Decimal<T> & operator /= (const T & x);
-    const Decimal<T> & operator %= (const T & x) { value %= x; return *this; }
+    const Decimal<T> & operator %= (const T & x);
 
-    template <typename U> const Decimal<T> & operator += (const Decimal<U> & x) { value += x.value; return *this; }
+    template <typename U> const Decimal<T> & operator += (const Decimal<U> & x);
-    template <typename U> const Decimal<T> & operator -= (const Decimal<U> & x) { value -= x.value; return *this; }
+    template <typename U> const Decimal<T> & operator -= (const Decimal<U> & x);
-    template <typename U> const Decimal<T> & operator *= (const Decimal<U> & x) { value *= x.value; return *this; }
+    template <typename U> const Decimal<T> & operator *= (const Decimal<U> & x);
-    template <typename U> const Decimal<T> & operator /= (const Decimal<U> & x) { value /= x.value; return *this; }
+    template <typename U> const Decimal<T> & operator /= (const Decimal<U> & x);
-    template <typename U> const Decimal<T> & operator %= (const Decimal<U> & x) { value %= x.value; return *this; }
+    template <typename U> const Decimal<T> & operator %= (const Decimal<U> & x);
 
     /// This is to avoid UB for sumWithOverflow()
-    void NO_SANITIZE_UNDEFINED addOverflow(const T & x) { value += x; }
+    void NO_SANITIZE_UNDEFINED addOverflow(const T & x);
 
     T value;
 };
 
-template <typename T> inline bool operator< (const Decimal<T> & x, const Decimal<T> & y) { return x.value < y.value; }
+#define DISPATCH(TYPE) extern template struct Decimal<TYPE>;
-template <typename T> inline bool operator> (const Decimal<T> & x, const Decimal<T> & y) { return x.value > y.value; }
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
-template <typename T> inline bool operator<= (const Decimal<T> & x, const Decimal<T> & y) { return x.value <= y.value; }
+#undef DISPATCH
-template <typename T> inline bool operator>= (const Decimal<T> & x, const Decimal<T> & y) { return x.value >= y.value; }
-template <typename T> inline bool operator== (const Decimal<T> & x, const Decimal<T> & y) { return x.value == y.value; }
-template <typename T> inline bool operator!= (const Decimal<T> & x, const Decimal<T> & y) { return x.value != y.value; }
 
-template <typename T> inline Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y) { return x.value + y.value; }
+#define DISPATCH(TYPE_T, TYPE_U) \
-template <typename T> inline Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y) { return x.value - y.value; }
+    extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator += (const Decimal<TYPE_U> & x); \
-template <typename T> inline Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y) { return x.value * y.value; }
+    extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator -= (const Decimal<TYPE_U> & x); \
-template <typename T> inline Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y) { return x.value / y.value; }
+    extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator *= (const Decimal<TYPE_U> & x); \
-template <typename T> inline Decimal<T> operator- (const Decimal<T> & x) { return -x.value; }
+    extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator /= (const Decimal<TYPE_U> & x); \
+    extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator %= (const Decimal<TYPE_U> & x);
+#define INVOKE(X) FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+template <typename T> bool operator< (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> bool operator> (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> bool operator<= (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> bool operator>= (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> bool operator== (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> bool operator!= (const Decimal<T> & x, const Decimal<T> & y);
+
+#define DISPATCH(TYPE) \
+extern template bool operator< (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template bool operator> (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template bool operator<= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template bool operator>= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template bool operator== (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template bool operator!= (const Decimal<TYPE> & x, const Decimal<TYPE> & y);
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
+
+template <typename T> Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> Decimal<T> operator- (const Decimal<T> & x);
+
+#define DISPATCH(TYPE) \
+extern template Decimal<TYPE> operator+ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template Decimal<TYPE> operator- (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template Decimal<TYPE> operator* (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template Decimal<TYPE> operator/ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template Decimal<TYPE> operator- (const Decimal<TYPE> & x);
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
+
+#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS
+#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE
 
 /// Distinguishable type to allow function resolution/deduction based on value type,
 /// but also relatively easy to convert to/from Decimal64.

base/base/extended_types.h

@@ -64,6 +64,44 @@ template <> struct is_arithmetic<UInt256> { static constexpr bool value = true;
 template <typename T>
 inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
 
+#define FOR_EACH_ARITHMETIC_TYPE(M) \
+    M(DataTypeDate) \
+    M(DataTypeDate32) \
+    M(DataTypeDateTime) \
+    M(DataTypeInt8) \
+    M(DataTypeUInt8) \
+    M(DataTypeInt16) \
+    M(DataTypeUInt16) \
+    M(DataTypeInt32) \
+    M(DataTypeUInt32) \
+    M(DataTypeInt64) \
+    M(DataTypeUInt64) \
+    M(DataTypeInt128) \
+    M(DataTypeUInt128) \
+    M(DataTypeInt256) \
+    M(DataTypeUInt256) \
+    M(DataTypeFloat32) \
+    M(DataTypeFloat64)
+
+#define FOR_EACH_ARITHMETIC_TYPE_PASS(M, X) \
+    M(DataTypeDate, X) \
+    M(DataTypeDate32, X) \
+    M(DataTypeDateTime, X) \
+    M(DataTypeInt8, X) \
+    M(DataTypeUInt8, X) \
+    M(DataTypeInt16, X) \
+    M(DataTypeUInt16, X) \
+    M(DataTypeInt32, X) \
+    M(DataTypeUInt32, X) \
+    M(DataTypeInt64, X) \
+    M(DataTypeUInt64, X) \
+    M(DataTypeInt128, X) \
+    M(DataTypeUInt128, X) \
+    M(DataTypeInt256, X) \
+    M(DataTypeUInt256, X) \
+    M(DataTypeFloat32, X) \
+    M(DataTypeFloat64, X)
+
 template <typename T>
 struct make_unsigned // NOLINT(readability-identifier-naming)
 {

docker/images.json

@@ -1,8 +1,12 @@
 {
-    "docker/packager/binary": {
+    "docker/packager/binary-builder": {
         "name": "clickhouse/binary-builder",
         "dependent": []
     },
+    "docker/packager/cctools": {
+        "name": "clickhouse/cctools",
+        "dependent": []
+    },
     "docker/test/compatibility/centos": {
         "name": "clickhouse/test-old-centos",
         "dependent": []
@@ -30,7 +34,6 @@
     "docker/test/util": {
         "name": "clickhouse/test-util",
         "dependent": [
-            "docker/packager/binary",
             "docker/test/base",
             "docker/test/fasttest"
         ]
@@ -67,7 +70,9 @@
     },
     "docker/test/fasttest": {
         "name": "clickhouse/fasttest",
-        "dependent": []
+        "dependent": [
+            "docker/packager/binary-builder"
+        ]
     },
     "docker/test/style": {
         "name": "clickhouse/style-test",

docker/packager/binary-builder/Dockerfile

@@ -1,43 +1,6 @@
 # docker build -t clickhouse/binary-builder .
 ARG FROM_TAG=latest
-FROM clickhouse/test-util:latest AS cctools
+FROM clickhouse/fasttest:$FROM_TAG
-# The cctools are built always from the clickhouse/test-util:latest and cached inline
-# Theoretically, it should improve rebuild speed significantly
-ENV CC=clang-${LLVM_VERSION}
-ENV CXX=clang++-${LLVM_VERSION}
-# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-# DO NOT PUT ANYTHING BEFORE THE NEXT TWO `RUN` DIRECTIVES
-# THE MOST HEAVY OPERATION MUST BE THE FIRST IN THE CACHE
-# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-# libtapi is required to support .tbh format from recent MacOS SDKs
-RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
-    && cd apple-libtapi \
-    && git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9 \
-    && INSTALLPREFIX=/cctools ./build.sh \
-    && ./install.sh \
-    && cd .. \
-    && rm -rf apple-libtapi
-
-# Build and install tools for cross-linking to Darwin (x86-64)
-# Build and install tools for cross-linking to Darwin (aarch64)
-RUN git clone https://github.com/tpoechtrager/cctools-port.git \
-    && cd cctools-port/cctools \
-    && git checkout 2a3e1c2a6ff54a30f898b70cfb9ba1692a55fad7 \
-    && ./configure --prefix=/cctools --with-libtapi=/cctools \
-        --target=x86_64-apple-darwin \
-    && make install -j$(nproc) \
-    && make clean \
-    && ./configure --prefix=/cctools --with-libtapi=/cctools \
-        --target=aarch64-apple-darwin \
-    && make install -j$(nproc) \
-    && cd ../.. \
-    && rm -rf cctools-port
-
-# !!!!!!!!!!!
-# END COMPILE
-# !!!!!!!!!!!
-
-FROM clickhouse/test-util:$FROM_TAG
 ENV CC=clang-${LLVM_VERSION}
 ENV CXX=clang++-${LLVM_VERSION}
 
@@ -110,7 +73,8 @@ RUN curl -Lo /usr/bin/clang-tidy-cache \
         "https://raw.githubusercontent.com/matus-chochlik/ctcache/$CLANG_TIDY_SHA1/clang-tidy-cache" \
     && chmod +x /usr/bin/clang-tidy-cache
 
-COPY --from=cctools /cctools /cctools
+# If the cctools is updated, then first build it in the CI, then update here in a different commit
+COPY --from=clickhouse/cctools:5a908f73878a /cctools /cctools
 
 RUN mkdir /workdir && chmod 777 /workdir
 WORKDIR /workdir

docker/packager/cctools/Dockerfile

@@ -0,0 +1,31 @@
+# This is a hack to significantly reduce the build time of the clickhouse/binary-builder
+# It's based on the assumption that we don't care of the cctools version so much
+# It event does not depend on the clickhouse/fasttest in the `docker/images.json`
+ARG FROM_TAG=latest
+FROM clickhouse/fasttest:$FROM_TAG
+
+ENV CC=clang-${LLVM_VERSION}
+ENV CXX=clang++-${LLVM_VERSION}
+
+RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
+    && cd apple-libtapi \
+    && git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9 \
+    && INSTALLPREFIX=/cctools ./build.sh \
+    && ./install.sh \
+    && cd .. \
+    && rm -rf apple-libtapi
+
+# Build and install tools for cross-linking to Darwin (x86-64)
+# Build and install tools for cross-linking to Darwin (aarch64)
+RUN git clone https://github.com/tpoechtrager/cctools-port.git \
+    && cd cctools-port/cctools \
+    && git checkout 2a3e1c2a6ff54a30f898b70cfb9ba1692a55fad7 \
+    && ./configure --prefix=/cctools --with-libtapi=/cctools \
+        --target=x86_64-apple-darwin \
+    && make install -j$(nproc) \
+    && make clean \
+    && ./configure --prefix=/cctools --with-libtapi=/cctools \
+        --target=aarch64-apple-darwin \
+    && make install -j$(nproc) \
+    && cd ../.. \
+    && rm -rf cctools-port

docker/packager/packager

@@ -1,16 +1,16 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-import subprocess
-import os
 import argparse
 import logging
+import os
+import subprocess
 import sys
 from pathlib import Path
 from typing import List, Optional
 
 SCRIPT_PATH = Path(__file__).absolute()
-IMAGE_TYPE = "binary"
+IMAGE_TYPE = "binary-builder"
-IMAGE_NAME = f"clickhouse/{IMAGE_TYPE}-builder"
+IMAGE_NAME = f"clickhouse/{IMAGE_TYPE}"
 
 
 class BuildException(Exception):

docker/test/fasttest/Dockerfile

@@ -6,9 +6,18 @@ FROM clickhouse/test-util:$FROM_TAG
 RUN apt-get update \
     && apt-get install \
         brotli \
+        clang-${LLVM_VERSION} \
+        clang-tidy-${LLVM_VERSION} \
+        cmake \
         expect \
         file \
+        libclang-${LLVM_VERSION}-dev \
+        libclang-rt-${LLVM_VERSION}-dev \
+        lld-${LLVM_VERSION} \
+        llvm-${LLVM_VERSION} \
+        llvm-${LLVM_VERSION}-dev \
         lsof \
+        ninja-build \
         odbcinst \
         psmisc \
         python3 \
@@ -26,14 +35,50 @@ RUN apt-get update \
 
 RUN pip3 install numpy==1.26.3 scipy==1.12.0 pandas==1.5.3 Jinja2==3.1.3
 
-ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
+# This symlink is required by gcc to find the lld linker
+RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
+# for external_symbolizer_path
+RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer
+# FIXME: workaround for "The imported target "merge-fdata" references the file" error
+# https://salsa.debian.org/pkg-llvm-team/llvm-toolchain/-/commit/992e52c0b156a5ba9c6a8a54f8c4857ddd3d371d
+RUN sed -i '/_IMPORT_CHECK_FILES_FOR_\(mlir-\|llvm-bolt\|merge-fdata\|MLIR\)/ {s|^|#|}' /usr/lib/llvm-${LLVM_VERSION}/lib/cmake/llvm/LLVMExports-*.cmake
 
-RUN mkdir -p /tmp/clickhouse-odbc-tmp \
+ARG CCACHE_VERSION=4.6.1
-  && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
+RUN mkdir /tmp/ccache \
-  && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
+    && cd /tmp/ccache \
-  && odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
+    && curl -L \
-  && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
+        -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz \
-  && rm -rf /tmp/clickhouse-odbc-tmp
+        -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz.asc \
+    && gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 5A939A71A46792CF57866A51996DDA075594ADB8 \
+    && gpg --verify ccache-4.6.1.tar.xz.asc \
+    && tar xf ccache-$CCACHE_VERSION.tar.xz \
+    && cd /tmp/ccache/ccache-$CCACHE_VERSION \
+    && cmake -DCMAKE_INSTALL_PREFIX=/usr \
+        -DCMAKE_BUILD_TYPE=None \
+        -DZSTD_FROM_INTERNET=ON \
+        -DREDIS_STORAGE_BACKEND=OFF \
+        -Wno-dev \
+        -B build \
+        -S . \
+    && make VERBOSE=1 -C build \
+    && make install -C build \
+    && cd / \
+    && rm -rf /tmp/ccache
+
+ARG TARGETARCH
+ARG SCCACHE_VERSION=v0.7.7
+ENV SCCACHE_IGNORE_SERVER_IO_ERROR=1
+# sccache requires a value for the region. So by default we use The Default Region
+ENV SCCACHE_REGION=us-east-1
+RUN arch=${TARGETARCH:-amd64} \
+  && case $arch in \
+    amd64) rarch=x86_64 ;; \
+    arm64) rarch=aarch64 ;; \
+  esac \
+  && curl -Ls "https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl.tar.gz" | \
+    tar xz -C /tmp \
+  && mv "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl/sccache" /usr/bin \
+  && rm "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl" -r
 
 # Give suid to gdb to grant it attach permissions
 # chmod 777 to make the container user independent

docker/test/fuzzer/run-fuzzer.sh

@@ -343,10 +343,9 @@ quit
         # which is confusing.
         task_exit_code=$fuzzer_exit_code
         echo "failure" > status.txt
-        { rg -ao "Found error:.*" fuzzer.log \
+        echo "Achtung!" > description.txt
-            || rg -ao "Exception:.*" fuzzer.log \
+        echo "Fuzzer went wrong with error code: ($fuzzer_exit_code). Its process died somehow when the server stayed alive. The server log probably won't tell you much so try to find information in other files." >>description.txt
-            || echo "Fuzzer failed ($fuzzer_exit_code). See the logs." ; } \
+        { rg -ao "Found error:.*" fuzzer.log || rg -ao "Exception:.*" fuzzer.log; } | tail -1 >>description.txt
-            | tail -1 > description.txt
     fi
 
     if test -f core.*; then

docker/test/sqllogic/Dockerfile

@@ -24,17 +24,18 @@ RUN pip3 install \
     deepdiff \
     sqlglot
 
-ARG odbc_repo="https://github.com/ClickHouse/clickhouse-odbc.git"
+ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz"
+
+RUN mkdir -p /tmp/clickhouse-odbc-tmp \
+  && cd /tmp/clickhouse-odbc-tmp \
+  && curl -L ${odbc_driver_url} | tar --strip-components=1 -xz clickhouse-odbc-1.1.6-Linux \
+  && mkdir /usr/local/lib64 -p \
+  && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib64/ \
+  && odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
+  && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
+  && sed -i 's"=libclickhouseodbc"=/usr/local/lib64/libclickhouseodbc"' /etc/odbcinst.ini \
+  && rm -rf /tmp/clickhouse-odbc-tmp
 
-RUN git clone --recursive ${odbc_repo} \
-    && mkdir -p /clickhouse-odbc/build \
-    && cmake -S /clickhouse-odbc -B /clickhouse-odbc/build \
-    && ls /clickhouse-odbc/build/driver \
-    && make -j 10 -C /clickhouse-odbc/build \
-    && ls /clickhouse-odbc/build/driver \
-    && mkdir -p /usr/local/lib64/ && cp /clickhouse-odbc/build/driver/lib*.so /usr/local/lib64/ \
-    && odbcinst -i -d -f /clickhouse-odbc/packaging/odbcinst.ini.sample \
-    && odbcinst -i -s -l -f /clickhouse-odbc/packaging/odbc.ini.sample
 
 ENV TZ=Europe/Amsterdam
 ENV MAX_RUN_TIME=9000

docker/test/stateless/Dockerfile

@@ -3,7 +3,7 @@
 ARG FROM_TAG=latest
 FROM clickhouse/test-base:$FROM_TAG
 
-ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
+ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz"
 
 # golang version 1.13 on Ubuntu 20 is enough for tests
 RUN apt-get update -y \
@@ -35,7 +35,6 @@ RUN apt-get update -y \
             sudo \
             tree \
             unixodbc \
-            wget \
             rustc \
             cargo \
             zstd \
@@ -50,10 +49,13 @@ RUN apt-get update -y \
 RUN pip3 install numpy==1.26.3 scipy==1.12.0 pandas==1.5.3 Jinja2==3.1.3 pyarrow==15.0.0
 
 RUN mkdir -p /tmp/clickhouse-odbc-tmp \
-   && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
+  && cd /tmp/clickhouse-odbc-tmp \
-   && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
+  && curl -L ${odbc_driver_url} | tar --strip-components=1 -xz clickhouse-odbc-1.1.6-Linux \
+  && mkdir /usr/local/lib64 -p \
+  && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib64/ \
   && odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
   && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
+  && sed -i 's"=libclickhouseodbc"=/usr/local/lib64/libclickhouseodbc"' /etc/odbcinst.ini \
   && rm -rf /tmp/clickhouse-odbc-tmp
 
 ENV TZ=Europe/Amsterdam
@@ -70,11 +72,11 @@ ARG TARGETARCH
 
 # Download Minio-related binaries
 RUN arch=${TARGETARCH:-amd64} \
-    && wget "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio.RELEASE.${MINIO_SERVER_VERSION}" -O ./minio \
+    && curl -L "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio.RELEASE.${MINIO_SERVER_VERSION}" -o ./minio \
-    && wget "https://dl.min.io/client/mc/release/linux-${arch}/archive/mc.RELEASE.${MINIO_CLIENT_VERSION}" -O ./mc \
+    && curl -L "https://dl.min.io/client/mc/release/linux-${arch}/archive/mc.RELEASE.${MINIO_CLIENT_VERSION}" -o ./mc \
     && chmod +x ./mc ./minio
 
-RUN wget --no-verbose 'https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
+RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
     && tar -xvf hadoop-3.3.1.tar.gz \
     && rm -rf hadoop-3.3.1.tar.gz
 

docker/test/style/Dockerfile

@@ -60,5 +60,4 @@ RUN arch=${TARGETARCH:-amd64} \
 
 
 COPY run.sh /
-COPY process_style_check_result.py /
 CMD ["/bin/bash", "/run.sh"]

docker/test/util/Dockerfile

@@ -41,20 +41,11 @@ RUN apt-get update \
         bash \
         bsdmainutils \
         build-essential \
-        clang-${LLVM_VERSION} \
-        clang-tidy-${LLVM_VERSION} \
-        cmake \
         gdb \
         git \
         gperf \
-        libclang-rt-${LLVM_VERSION}-dev \
-        lld-${LLVM_VERSION} \
-        llvm-${LLVM_VERSION} \
-        llvm-${LLVM_VERSION}-dev \
-        libclang-${LLVM_VERSION}-dev \
         moreutils \
         nasm \
-        ninja-build \
         pigz \
         rename \
         software-properties-common \
@@ -63,49 +54,4 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
-# This symlink is required by gcc to find the lld linker
-RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
-# for external_symbolizer_path
-RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer
-# FIXME: workaround for "The imported target "merge-fdata" references the file" error
-# https://salsa.debian.org/pkg-llvm-team/llvm-toolchain/-/commit/992e52c0b156a5ba9c6a8a54f8c4857ddd3d371d
-RUN sed -i '/_IMPORT_CHECK_FILES_FOR_\(mlir-\|llvm-bolt\|merge-fdata\|MLIR\)/ {s|^|#|}' /usr/lib/llvm-${LLVM_VERSION}/lib/cmake/llvm/LLVMExports-*.cmake
-
-ARG CCACHE_VERSION=4.6.1
-RUN mkdir /tmp/ccache \
-    && cd /tmp/ccache \
-    && curl -L \
-        -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz \
-        -O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz.asc \
-    && gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 5A939A71A46792CF57866A51996DDA075594ADB8 \
-    && gpg --verify ccache-4.6.1.tar.xz.asc \
-    && tar xf ccache-$CCACHE_VERSION.tar.xz \
-    && cd /tmp/ccache/ccache-$CCACHE_VERSION \
-    && cmake -DCMAKE_INSTALL_PREFIX=/usr \
-        -DCMAKE_BUILD_TYPE=None \
-        -DZSTD_FROM_INTERNET=ON \
-        -DREDIS_STORAGE_BACKEND=OFF \
-        -Wno-dev \
-        -B build \
-        -S . \
-    && make VERBOSE=1 -C build \
-    && make install -C build \
-    && cd / \
-    && rm -rf /tmp/ccache
-
-ARG TARGETARCH
-ARG SCCACHE_VERSION=v0.5.4
-ENV SCCACHE_IGNORE_SERVER_IO_ERROR=1
-# sccache requires a value for the region. So by default we use The Default Region
-ENV SCCACHE_REGION=us-east-1
-RUN arch=${TARGETARCH:-amd64} \
-  && case $arch in \
-    amd64) rarch=x86_64 ;; \
-    arm64) rarch=aarch64 ;; \
-  esac \
-  && curl -Ls "https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl.tar.gz" | \
-    tar xz -C /tmp \
-  && mv "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl/sccache" /usr/bin \
-  && rm "/tmp/sccache-$SCCACHE_VERSION-$rarch-unknown-linux-musl" -r
-
 COPY process_functional_tests_result.py /

docs/en/operations/settings/settings.md

@@ -3954,6 +3954,7 @@ Possible values:
 - `none` — Is similar to throw, but distributed DDL query returns no result set.
 - `null_status_on_timeout` — Returns `NULL` as execution status in some rows of result set instead of throwing `TIMEOUT_EXCEEDED` if query is not finished on the corresponding hosts.
 - `never_throw` — Do not throw `TIMEOUT_EXCEEDED` and do not rethrow exceptions if query has failed on some hosts.
+- `none_only_active` - similar to `none`, but doesn't wait for inactive replicas of the `Replicated` database. Note: with this mode it's impossible to figure out that the query was not executed on some replica and will be executed in background.
 - `null_status_on_timeout_only_active` — similar to `null_status_on_timeout`, but doesn't wait for inactive replicas of the `Replicated` database
 - `throw_only_active` — similar to `throw`, but doesn't wait for inactive replicas of the `Replicated` database
 

docs/en/operations/utilities/clickhouse-local.md

@@ -201,12 +201,12 @@ Arguments:
 
 - `-S`, `--structure` — table structure for input data.
 - `--input-format` — input format, `TSV` by default.
-- `-f`, `--file` — path to data, `stdin` by default.
+- `-F`, `--file` — path to data, `stdin` by default.
 - `-q`, `--query` — queries to execute with `;` as delimiter. `--query` can be specified multiple times, e.g. `--query "SELECT 1" --query "SELECT 2"`. Cannot be used simultaneously with `--queries-file`.
 - `--queries-file` - file path with queries to execute. `--queries-file` can be specified multiple times, e.g. `--query queries1.sql --query queries2.sql`. Cannot be used simultaneously with `--query`.
 - `--multiquery, -n` – If specified, multiple queries separated by semicolons can be listed after the `--query` option. For convenience, it is also possible to omit `--query` and pass the queries directly after `--multiquery`.
 - `-N`, `--table` — table name where to put output data, `table` by default.
-- `--format`, `--output-format` — output format, `TSV` by default.
+- `-f`, `--format`, `--output-format` — output format, `TSV` by default.
 - `-d`, `--database` — default database, `_local` by default.
 - `--stacktrace` — whether to dump debug output in case of exception.
 - `--echo` — print query before execution.

docs/en/sql-reference/functions/date-time-functions.md

@@ -394,8 +394,7 @@ Result:
 
 ## toYear
 
-Converts a date or date with time to the year number (AD) as `UInt16` value.
+Returns the year component (AD) of a date or date with time.
-
 
 **Syntax**
 
@@ -431,7 +430,7 @@ Result:
 
 ## toQuarter
 
-Converts a date or date with time to the quarter number (1-4) as `UInt8` value.
+Returns the quarter (1-4) of a date or date with time.
 
 **Syntax**
 
@@ -465,10 +464,9 @@ Result:
 └──────────────────────────────────────────────┘
 ```
 
-
 ## toMonth
 
-Converts a date or date with time to the month number (1-12) as `UInt8` value.
+Returns the month component (1-12) of a date or date with time.
 
 **Syntax**
 
@@ -504,7 +502,7 @@ Result:
 
 ## toDayOfYear
 
-Converts a date or date with time to the number of the day of the year (1-366) as `UInt16` value.
+Returns the number of the day within the year (1-366) of a date or date with time.
 
 **Syntax**
 
@@ -540,7 +538,7 @@ Result:
 
 ## toDayOfMonth
 
-Converts a date or date with time to the number of the day in the month (1-31) as `UInt8` value.
+Returns the number of the day within the month (1-31) of a date or date with time.
 
 **Syntax**
 
@@ -576,7 +574,7 @@ Result:
 
 ## toDayOfWeek
 
-Converts a date or date with time to the number of the day in the week as `UInt8` value.
+Returns the number of the day within the week of a date or date with time.
 
 The two-argument form of `toDayOfWeek()` enables you to specify whether the week starts on Monday or Sunday, and whether the return value should be in the range from 0 to 6 or 1 to 7. If the mode argument is omitted, the default mode is 0. The time zone of the date can be specified as the third argument.
 
@@ -627,7 +625,7 @@ Result:
 
 ## toHour
 
-Converts a date with time to the number of the hour in 24-hour time (0-23) as `UInt8` value.
+Returns the hour component (0-24) of a date with time.
 
 Assumes that if clocks are moved ahead, it is by one hour and occurs at 2 a.m., and if clocks are moved back, it is by one hour and occurs at 3 a.m. (which is not always exactly when it occurs - it depends on the timezone).
 
@@ -641,7 +639,7 @@ Alias: `HOUR`
 
 **Arguments**
 
-- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
 
 **Returned value**
 
@@ -665,7 +663,7 @@ Result:
 
 ## toMinute
 
-Converts a date with time to the number of the minute of the hour (0-59) as `UInt8` value.
+Returns the minute component (0-59) a date with time.
 
 **Syntax**
 
@@ -677,7 +675,7 @@ Alias: `MINUTE`
 
 **Arguments**
 
-- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
 
 **Returned value**
 
@@ -701,7 +699,7 @@ Result:
 
 ## toSecond
 
-Converts a date with time to the second in the minute (0-59) as `UInt8` value. Leap seconds are not considered.
+Returns the second component (0-59) of a date with time. Leap seconds are not considered.
 
 **Syntax**
 
@@ -713,7 +711,7 @@ Alias: `SECOND`
 
 **Arguments**
 
-- `value` - a [Date](../data-types/date.md), [Date32](../data-types/date32.md), [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+- `value` - a [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
 
 **Returned value**
 
@@ -735,6 +733,40 @@ Result:
 └─────────────────────────────────────────────┘
 ```
 
+## toMillisecond
+
+Returns the millisecond component (0-999) of a date with time.
+
+**Syntax**
+
+```sql
+toMillisecond(value)
+```
+
+*Arguments**
+
+- `value` - [DateTime](../data-types/datetime.md) or [DateTime64](../data-types/datetime64.md)
+
+Alias: `MILLISECOND`
+
+```sql
+SELECT toMillisecond(toDateTime64('2023-04-21 10:20:30.456', 3))
+```
+
+Result:
+
+```response
+┌──toMillisecond(toDateTime64('2023-04-21 10:20:30.456', 3))─┐
+│                                                        456 │
+└────────────────────────────────────────────────────────────┘
+```
+
+**Returned value**
+
+- The millisecond in the minute (0 - 59) of the given date/time
+
+Type: `UInt16`
+
 ## toUnixTimestamp
 
 Converts a string, a date or a date with time to the [Unix Timestamp](https://en.wikipedia.org/wiki/Unix_time) in `UInt32` representation.

docs/en/sql-reference/statements/grant.md

@@ -202,6 +202,13 @@ Hierarchy of privileges:
     - `S3`
 - [dictGet](#grant-dictget)
 - [displaySecretsInShowAndSelect](#grant-display-secrets)
+- [NAMED COLLECTION ADMIN](#grant-named-collection-admin)
+    - `CREATE NAMED COLLECTION`
+    - `DROP NAMED COLLECTION`
+    - `ALTER NAMED COLLECTION`
+    - `SHOW NAMED COLLECTIONS`
+    - `SHOW NAMED COLLECTIONS SECRETS`
+    - `NAMED COLLECTION`
 
 Examples of how this hierarchy is treated:
 
@@ -498,6 +505,25 @@ and
 [`format_display_secrets_in_show_and_select` format setting](../../operations/settings/formats#format_display_secrets_in_show_and_select)
 are turned on.
 
+### NAMED COLLECTION ADMIN
+
+Allows a certain operation on a specified named collection. Before version 23.7 it was called NAMED COLLECTION CONTROL, and after 23.7 NAMED COLLECTION ADMIN was added and NAMED COLLECTION CONTROL is preserved as an alias.
+
+- `NAMED COLLECTION ADMIN`. Level: `NAMED_COLLECTION`. Aliases: `NAMED COLLECTION CONTROL`
+    - `CREATE NAMED COLLECTION`. Level: `NAMED_COLLECTION`
+    - `DROP NAMED COLLECTION`. Level: `NAMED_COLLECTION`
+    - `ALTER NAMED COLLECTION`. Level: `NAMED_COLLECTION`
+    - `SHOW NAMED COLLECTIONS`. Level: `NAMED_COLLECTION`. Aliases: `SHOW NAMED COLLECTIONS`
+    - `SHOW NAMED COLLECTIONS SECRETS`. Level: `NAMED_COLLECTION`. Aliases: `SHOW NAMED COLLECTIONS SECRETS`
+    - `NAMED COLLECTION`. Level: `NAMED_COLLECTION`. Aliases: `NAMED COLLECTION USAGE, USE NAMED COLLECTION`
+
+Unlike all other grants (CREATE, DROP, ALTER, SHOW) grant NAMED COLLECTION was added only in 23.7, while all others were added earlier - in 22.12.
+
+**Examples**
+
+Assuming a named collection is called abc, we grant privilege CREATE NAMED COLLECTION to user john.
+- `GRANT CREATE NAMED COLLECTION ON abc TO john`
+
 ### ALL
 
 Grants all the privileges on regulated entity to a user account or a role.

docs/en/sql-reference/window-functions/index.md

@@ -5,7 +5,12 @@ sidebar_label: Window Functions
 title: Window Functions
 ---
 
-ClickHouse supports the standard grammar for defining windows and window functions. The following features are currently supported:
+Windows functions let you perform calculations across a set of rows that are related to the current row.
+Some of the calculations that you can do are similar to those that can be done with an aggregate function, but a window function doesn't cause rows to be grouped into a single output - the individual rows are still returned.
+
+## Standard Window Functions
+
+ClickHouse supports the standard grammar for defining windows and window functions. The table below indicates whether a feature is currently supported.
 
 | Feature                                                                            | Support or workaround                                                                                                                                                                       |
 |------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -25,6 +30,8 @@ ClickHouse supports the standard grammar for defining windows and window functio
 
 ## ClickHouse-specific Window Functions
 
+There are also the following window function that's specific to ClickHouse:
+
 ### nonNegativeDerivative(metric_column, timestamp_column[, INTERVAL X UNITS])
 
 Finds non-negative derivative for given `metric_column` by `timestamp_column`. 
@@ -33,40 +40,6 @@ The computed value is the following for each row:
 - `0` for 1st row,
 - ${metric_i - metric_{i-1} \over timestamp_i - timestamp_{i-1}}  * interval$ for $i_th$ row.
 
-## References
-
-### GitHub Issues
-
-The roadmap for the initial support of window functions is [in this issue](https://github.com/ClickHouse/ClickHouse/issues/18097).
-
-All GitHub issues related to window functions have the [comp-window-functions](https://github.com/ClickHouse/ClickHouse/labels/comp-window-functions) tag.
-
-### Tests
-
-These tests contain the examples of the currently supported grammar:
-
-https://github.com/ClickHouse/ClickHouse/blob/master/tests/performance/window_functions.xml
-
-https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/01591_window_functions.sql
-
-### Postgres Docs
-
-https://www.postgresql.org/docs/current/sql-select.html#SQL-WINDOW
-
-https://www.postgresql.org/docs/devel/sql-expressions.html#SYNTAX-WINDOW-FUNCTIONS
-
-https://www.postgresql.org/docs/devel/functions-window.html
-
-https://www.postgresql.org/docs/devel/tutorial-window.html
-
-### MySQL Docs
-
-https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html
-
-https://dev.mysql.com/doc/refman/8.0/en/window-functions-usage.html
-
-https://dev.mysql.com/doc/refman/8.0/en/window-functions-frames.html
-
 ## Syntax
 
 ```text
@@ -80,20 +53,7 @@ WINDOW window_name as ([[PARTITION BY grouping_column] [ORDER BY sorting_column]
 - `PARTITION BY` - defines how to break a resultset into groups.
 - `ORDER BY` - defines how to order rows inside the group during calculation aggregate_function.
 - `ROWS or RANGE` - defines bounds of a frame, aggregate_function is calculated within a frame.
-- `WINDOW` - allows to reuse a window definition with multiple expressions.
+- `WINDOW` - allows multiple expressions to use the same window definition.
-
-### Functions
-
-These functions can be used only as a window function.
-
-- `row_number()` - Number the current row within its partition starting from 1.
-- `first_value(x)` - Return the first non-NULL value evaluated within its ordered frame.
-- `last_value(x)` -	Return the last non-NULL value evaluated within its ordered frame.
-- `nth_value(x, offset)` - Return the first non-NULL value evaluated against the nth row (offset) in its ordered frame.
-- `rank()` - Rank the current row within its partition with gaps.
-- `dense_rank()` - Rank the current row within its partition without gaps.
-- `lagInFrame(x)` - Return a value evaluated at the row that is at a specified physical offset row before the current row within the ordered frame.
-- `leadInFrame(x)` - Return a value evaluated at the row that is offset rows after the current row within the ordered frame.
 
 ```text
       PARTITION
@@ -112,8 +72,23 @@ These functions can be used only as a window function.
 └─────────────────┘  <--- UNBOUNDED FOLLOWING (END of the PARTITION)
 ```
 
+### Functions
+
+These functions can be used only as a window function.
+
+- `row_number()` - Number the current row within its partition starting from 1.
+- `first_value(x)` - Return the first non-NULL value evaluated within its ordered frame.
+- `last_value(x)` -	Return the last non-NULL value evaluated within its ordered frame.
+- `nth_value(x, offset)` - Return the first non-NULL value evaluated against the nth row (offset) in its ordered frame.
+- `rank()` - Rank the current row within its partition with gaps.
+- `dense_rank()` - Rank the current row within its partition without gaps.
+- `lagInFrame(x)` - Return a value evaluated at the row that is at a specified physical offset row before the current row within the ordered frame.
+- `leadInFrame(x)` - Return a value evaluated at the row that is offset rows after the current row within the ordered frame.
+
 ## Examples
 
+Let's have a look at some examples of how window functions can be used.
+
 ```sql
 CREATE TABLE wf_partition
 (
@@ -589,6 +564,41 @@ ORDER BY
 └──────────────┴─────────────────────┴───────┴─────────────────────────┘
 ```
 
+## References
+
+### GitHub Issues
+
+The roadmap for the initial support of window functions is [in this issue](https://github.com/ClickHouse/ClickHouse/issues/18097).
+
+All GitHub issues related to window functions have the [comp-window-functions](https://github.com/ClickHouse/ClickHouse/labels/comp-window-functions) tag.
+
+### Tests
+
+These tests contain the examples of the currently supported grammar:
+
+https://github.com/ClickHouse/ClickHouse/blob/master/tests/performance/window_functions.xml
+
+https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/01591_window_functions.sql
+
+### Postgres Docs
+
+https://www.postgresql.org/docs/current/sql-select.html#SQL-WINDOW
+
+https://www.postgresql.org/docs/devel/sql-expressions.html#SYNTAX-WINDOW-FUNCTIONS
+
+https://www.postgresql.org/docs/devel/functions-window.html
+
+https://www.postgresql.org/docs/devel/tutorial-window.html
+
+### MySQL Docs
+
+https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html
+
+https://dev.mysql.com/doc/refman/8.0/en/window-functions-usage.html
+
+https://dev.mysql.com/doc/refman/8.0/en/window-functions-frames.html
+
+
 ## Related Content
 
 - Blog: [Working with time series data in ClickHouse](https://clickhouse.com/blog/working-with-time-series-data-and-functions-ClickHouse)

docs/zh/operations/settings/settings.md

@@ -649,11 +649,22 @@ log_query_threads=1
 
 ## max_query_size {#settings-max_query_size}
 
-查询的最大部分，可以被带到RAM用于使用SQL解析器进行解析。
+SQL 解析器解析的查询字符串的最大字节数。 INSERT 查询的 VALUES 子句中的数据由单独的流解析器（消耗 O(1) RAM）处理，并且不受此限制的影响。
-插入查询还包含由单独的流解析器（消耗O(1)RAM）处理的插入数据，这些数据不包含在此限制中。
 
 默认值：256KiB。
 
+
+## max_parser_depth {#max_parser_depth}
+
+限制递归下降解析器中的最大递归深度。允许控制堆栈大小。
+
+可能的值：
+
+- 正整数。
+- 0 — 递归深度不受限制。
+
+默认值：1000。
+
 ## interactive_delay {#interactive-delay}
 
 以微秒为单位的间隔，用于检查请求执行是否已被取消并发送进度。
@@ -1064,6 +1075,28 @@ ClickHouse生成异常
 
 默认值：0。
 
+## optimize_functions_to_subcolumns {#optimize_functions_to_subcolumns}
+
+启用或禁用通过将某些函数转换为读取子列的优化。这减少了要读取的数据量。
+
+这些函数可以转化为：
+
+- [length](../../sql-reference/functions/array-functions.md/#array_functions-length) 读取 [size0](../../sql-reference/data-types/array.md/#array-size）子列。
+- [empty](../../sql-reference/functions/array-functions.md/#empty函数) 读取 [size0](../../sql-reference/data-types/array.md/#array-size）子列。
+- [notEmpty](../../sql-reference/functions/array-functions.md/#notempty函数) 读取 [size0](../../sql-reference/data-types/array.md/#array-size）子列。
+- [isNull](../../sql-reference/operators/index.md#operator-is-null) 读取 [null](../../sql-reference/data-types/nullable. md/#finding-null) 子列。
+- [isNotNull](../../sql-reference/operators/index.md#is-not-null) 读取 [null](../../sql-reference/data-types/nullable. md/#finding-null) 子列。
+- [count](../../sql-reference/aggregate-functions/reference/count.md) 读取 [null](../../sql-reference/data-types/nullable.md/#finding-null) 子列。
+- [mapKeys](../../sql-reference/functions/tuple-map-functions.mdx/#mapkeys) 读取 [keys](../../sql-reference/data-types/map.md/#map-subcolumns) 子列。
+- [mapValues](../../sql-reference/functions/tuple-map-functions.mdx/#mapvalues) 读取 [values](../../sql-reference/data-types/map.md/#map-subcolumns) 子列。
+
+可能的值：
+
+- 0 — 禁用优化。
+- 1 — 优化已启用。
+
+默认值：`0`。
+
 ## distributed_replica_error_half_life {#settings-distributed_replica_error_half_life}
 
 -   类型：秒

docs/zh/sql-reference/data-types/array.md

@@ -1,7 +1,7 @@
 ---
 slug: /zh/sql-reference/data-types/array
 ---
-# 阵列(T) {#data-type-array}
+# 数组(T) {#data-type-array}
 
 由 `T` 类型元素组成的数组。
 
@@ -66,3 +66,27 @@ SELECT array(1, 'a')
 Received exception from server (version 1.1.54388):
 Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: There is no supertype for types UInt8, String because some of them are String/FixedString and some of them are not.
 ```
+
+## 数组大小 {#array-size}
+
+可以使用 `size0` 子列找到数组的大小，而无需读取整个列。对于多维数组，您可以使用 `sizeN-1`，其中 `N` 是所需的维度。
+
+**例子**
+
+SQL查询：
+
+```sql
+CREATE TABLE t_arr (`arr` Array(Array(Array(UInt32)))) ENGINE = MergeTree ORDER BY tuple();
+
+INSERT INTO t_arr VALUES ([[[12, 13, 0, 1],[12]]]);
+
+SELECT arr.size0, arr.size1, arr.size2 FROM t_arr;
+```
+
+结果：
+
+``` text
+┌─arr.size0─┬─arr.size1─┬─arr.size2─┐
+│         1 │ [2]       │ [[4,1]]   │
+└───────────┴───────────┴───────────┘
+```

docs/zh/sql-reference/data-types/nullable.md

@@ -20,6 +20,33 @@ slug: /zh/sql-reference/data-types/nullable
 
 掩码文件中的条目允许ClickHouse区分每个表行的对应数据类型的«NULL»和默认值由于有额外的文件，«Nullable»列比普通列消耗更多的存储空间
 
+## null子列 {#finding-null}
+
+通过使用 `null` 子列可以在列中查找 `NULL` 值，而无需读取整个列。如果对应的值为 `NULL`，则返回 `1`，否则返回 `0`。
+
+**示例**
+
+SQL查询:
+
+``` sql
+CREATE TABLE nullable (`n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple();
+
+INSERT INTO nullable VALUES (1) (NULL) (2) (NULL);
+
+SELECT n.null FROM nullable;
+```
+
+结果:
+
+``` text
+┌─n.null─┐
+│      0 │
+│      1 │
+│      0 │
+│      1 │
+└────────┘
+```
+
 ## 用法示例 {#yong-fa-shi-li}
 
 ``` sql

programs/copier/Internals.cpp

@@ -259,7 +259,7 @@ ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std
     res.is_remote = 1;
     for (const auto & replica : replicas)
     {
-        if (isLocalAddress(DNSResolver::instance().resolveHost(replica.host_name)))
+        if (isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(replica.host_name).front()))
         {
             res.is_remote = 0;
             break;

rust/Cargo.lock

@@ -6,6 +6,7 @@ version = 3
 name = "_ch_rust_prql"
 version = "0.1.0"
 dependencies = [
+ "anstream",
  "prqlc",
  "serde_json",
 ]
@@ -698,9 +699,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.7.1"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
+checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
 dependencies = [
  "adler",
 ]
@@ -751,9 +752,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.32.1"
+version = "0.32.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
+checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
 dependencies = [
  "memchr",
 ]

rust/prql/Cargo.toml

@@ -4,6 +4,7 @@ name = "_ch_rust_prql"
 version = "0.1.0"
 
 [dependencies]
+anstream = {version = "0.6.12"}
 prqlc = {version = "0.11.3", default-features = false}
 serde_json = "1.0"
 

rust/prql/src/lib.rs

@@ -39,6 +39,11 @@ pub unsafe extern "C" fn prql_to_sql_impl(
     };
 
     if let Ok(sql_str) = prqlc::compile(&query_str, &opts) {
+        // NOTE: Over at PRQL we're considering to un-deprecate & re-enable the
+        // `color: false` option. If that happens, we can remove the `strip_str`
+        // here, which strips color codes from the output.
+        use anstream::adapter::strip_str;
+        let sql_str = strip_str(&sql_str).to_string();
         set_output(sql_str, out, out_size);
         0
     } else {
@@ -54,17 +59,50 @@ pub unsafe extern "C" fn prql_to_sql(
     out: *mut *mut u8,
     out_size: *mut u64,
 ) -> i64 {
-    let ret = panic::catch_unwind(|| {
-        return prql_to_sql_impl(query, size, out, out_size);
-    });
-    return match ret {
     // NOTE: using cxxbridge we can return proper Result<> type.
-        Err(_err) => 1,
+    panic::catch_unwind(|| prql_to_sql_impl(query, size, out, out_size)).unwrap_or_else(|_| {
-        Ok(res) => res,
+        set_output("prqlc panicked".to_string(), out, out_size);
-    }
+        1
+    })
 }
 
 #[no_mangle]
 pub unsafe extern "C" fn prql_free_pointer(ptr_to_free: *mut u8) {
     std::mem::drop(CString::from_raw(ptr_to_free as *mut c_char));
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::ffi::{CStr, CString};
+
+    /// A test helper to offer a rust interface to the C bindings
+    fn run_compile(query: &str) -> (String, i64) {
+        let query_cstr = CString::new(query).unwrap();
+        let query_ptr = query_cstr.as_ptr() as *const u8;
+        let query_size = query_cstr.to_bytes_with_nul().len() as u64 - 1; // Excluding the null terminator
+
+        let mut out: *mut u8 = std::ptr::null_mut();
+        let mut out_size = 0_u64;
+
+        unsafe {
+            let success = prql_to_sql(query_ptr, query_size, &mut out, &mut out_size);
+            let output = CStr::from_ptr(out as *const i8)
+                .to_str()
+                .unwrap()
+                .to_string();
+            prql_free_pointer(out);
+            (output, success)
+        }
+    }
+
+    #[test]
+    fn test_prql_to_sql() {
+        assert!(run_compile("from x").0.contains("SELECT"));
+        assert!(run_compile("asdf").1 == 1);
+        // In prqlc 0.11.3, this is a panic, so that allows us to test that the
+        // panic is caught. When we upgrade prqlc, it won't be a panic any
+        // longer.
+        assert!(run_compile("x -> y").1 == 1);
+    }
+}

src/Access/Common/AllowedClientHosts.cpp

@@ -55,7 +55,7 @@ namespace
     {
         IPAddress addr_v6 = toIPv6(address);
 
-        auto host_addresses = DNSResolver::instance().resolveHostAll(host);
+        auto host_addresses = DNSResolver::instance().resolveHostAllInOriginOrder(host);
 
         for (const auto & addr : host_addresses)
         {

src/Access/Common/QuotaDefs.cpp

@@ -49,71 +49,135 @@ String QuotaTypeInfo::valueToStringWithName(QuotaValue value) const
 
 const QuotaTypeInfo & QuotaTypeInfo::get(QuotaType type)
 {
-    static constexpr auto make_info = [](const char * raw_name_, UInt64 output_denominator_)
+    static constexpr auto make_info = [](const char * raw_name_, String current_usage_description_, String max_allowed_usage_description_, UInt64 output_denominator_)
     {
         String init_name = raw_name_;
         boost::to_lower(init_name);
         String init_keyword = raw_name_;
         boost::replace_all(init_keyword, "_", " ");
         bool init_output_as_float = (output_denominator_ != 1);
-        return QuotaTypeInfo{raw_name_, std::move(init_name), std::move(init_keyword), init_output_as_float, output_denominator_};
+        return QuotaTypeInfo
+        {
+            .raw_name = raw_name_,
+            .name = std::move(init_name),
+            .keyword = std::move(init_keyword),
+            .current_usage_description = std::move(current_usage_description_),
+            .max_allowed_usage_description = std::move(max_allowed_usage_description_),
+            .output_as_float = init_output_as_float,
+            .output_denominator = output_denominator_
+        };
     };
 
     switch (type)
     {
         case QuotaType::QUERIES:
         {
-            static const auto info = make_info("QUERIES", 1);
+            static const auto info = make_info(
+                "QUERIES",
+                "The current number of executed queries.",
+                "The maximum allowed number of queries of all types allowed to be executed.",
+                1
+            );
             return info;
         }
         case QuotaType::QUERY_SELECTS:
         {
-            static const auto info = make_info("QUERY_SELECTS", 1);
+            static const auto info = make_info(
+                "QUERY_SELECTS",
+                "The current number of executed SELECT queries.",
+                "The maximum allowed number of SELECT queries allowed to be executed.",
+                1
+            );
             return info;
         }
         case QuotaType::QUERY_INSERTS:
         {
-            static const auto info = make_info("QUERY_INSERTS", 1);
+            static const auto info = make_info(
+                "QUERY_INSERTS",
+                "The current number of executed INSERT queries.",
+                "The maximum allowed number of INSERT queries allowed to be executed.",
+                1
+            );
             return info;
         }
         case QuotaType::ERRORS:
         {
-            static const auto info = make_info("ERRORS", 1);
+            static const auto info = make_info(
+                "ERRORS",
+                "The current number of queries resulted in an error.",
+                "The maximum number of queries resulted in an error allowed within the specified period of time.",
+                1
+            );
             return info;
         }
         case QuotaType::RESULT_ROWS:
         {
-            static const auto info = make_info("RESULT_ROWS", 1);
+            static const auto info = make_info(
+                "RESULT_ROWS",
+                "The current total number of rows in the result set of all queries within the current period of time.",
+                "The maximum total number of rows in the result set of all queries allowed within the specified period of time.",
+                1
+            );
             return info;
         }
         case QuotaType::RESULT_BYTES:
         {
-            static const auto info = make_info("RESULT_BYTES", 1);
+            static const auto info = make_info(
+                "RESULT_BYTES",
+                "The current total number of bytes in the result set of all queries within the current period of time.",
+                "The maximum total number of bytes in the result set of all queries allowed within the specified period of time.",
+                1
+            );
             return info;
         }
         case QuotaType::READ_ROWS:
         {
-            static const auto info = make_info("READ_ROWS", 1);
+            static const auto info = make_info(
+                "READ_ROWS",
+                "The current total number of rows read during execution of all queries within the current period of time.",
+                "The maximum number of rows to read during execution of all queries allowed within the specified period of time.",
+                1
+            );
             return info;
         }
         case QuotaType::READ_BYTES:
         {
-            static const auto info = make_info("READ_BYTES", 1);
+            static const auto info = make_info(
+                "READ_BYTES",
+                "The current total number of bytes read during execution of all queries within the current period of time.",
+                "The maximum number of bytes to read during execution of all queries allowed within the specified period of time.",
+                1
+            );
             return info;
         }
         case QuotaType::EXECUTION_TIME:
         {
-            static const auto info = make_info("EXECUTION_TIME", 1000000000 /* execution_time is stored in nanoseconds */);
+            static const auto info = make_info(
+                "EXECUTION_TIME",
+                "The current total amount of time (in nanoseconds) spent to execute queries within the current period of time",
+                "The maximum amount of time (in nanoseconds) allowed for all queries to execute within the specified period of time",
+                1000000000 /* execution_time is stored in nanoseconds */
+            );
             return info;
         }
         case QuotaType::WRITTEN_BYTES:
         {
-            static const auto info = make_info("WRITTEN_BYTES", 1);
+            static const auto info = make_info(
+                "WRITTEN_BYTES",
+                "The current total number of bytes written during execution of all queries within the current period of time.",
+                "The maximum number of bytes to written during execution of all queries allowed within the specified period of time.",
+                1
+            );
             return info;
         }
         case QuotaType::FAILED_SEQUENTIAL_AUTHENTICATIONS:
         {
-            static const auto info = make_info("FAILED_SEQUENTIAL_AUTHENTICATIONS", 1);
+            static const auto info = make_info(
+                "FAILED_SEQUENTIAL_AUtheNTICATIONS",
+                "The current number of consecutive authentication failures within the current period of time.",
+                "The maximum number of consecutive authentication failures allowed within the specified period of time.",
+                1
+            );
             return info;
         }
         case QuotaType::MAX: break;

src/Access/Common/QuotaDefs.h

@@ -33,6 +33,8 @@ struct QuotaTypeInfo
     const char * const raw_name = "";
     const String name;    /// Lowercased with underscores, e.g. "result_rows".
     const String keyword; /// Uppercased with spaces, e.g. "RESULT ROWS".
+    const String current_usage_description;
+    const String max_allowed_usage_description;
     const bool output_as_float = false;
     const UInt64 output_denominator = 1;
     String valueToString(QuotaValue value) const;

src/Access/Common/RowPolicyDefs.cpp

@@ -33,7 +33,7 @@ String toString(RowPolicyFilterType type)
 
 const RowPolicyFilterTypeInfo & RowPolicyFilterTypeInfo::get(RowPolicyFilterType type_)
 {
-    static constexpr auto make_info = [](const char * raw_name_)
+    static constexpr auto make_info = [](const char * raw_name_, const String & comment_)
     {
         String init_name = raw_name_;
         boost::to_lower(init_name);
@@ -41,14 +41,17 @@ const RowPolicyFilterTypeInfo & RowPolicyFilterTypeInfo::get(RowPolicyFilterType
         String init_command = init_name.substr(0, underscore_pos);
         boost::to_upper(init_command);
         bool init_is_check = (std::string_view{init_name}.substr(underscore_pos + 1) == "check");
-        return RowPolicyFilterTypeInfo{raw_name_, std::move(init_name), std::move(init_command), init_is_check};
+        return RowPolicyFilterTypeInfo{raw_name_, std::move(init_name), std::move(init_command), comment_, init_is_check};
     };
 
     switch (type_)
     {
         case RowPolicyFilterType::SELECT_FILTER:
         {
-            static const auto info = make_info("SELECT_FILTER");
+            static const auto info = make_info(
+                "SELECT_FILTER",
+                "Expression which is used for filtering in SELECT queries."
+            );
             return info;
         }
 #if 0 /// Row-level security for INSERT, UPDATE, DELETE is not implemented yet.

src/Access/Common/RowPolicyDefs.h

@@ -52,6 +52,7 @@ struct RowPolicyFilterTypeInfo
     const char * const raw_name;
     const String name;    /// Lowercased with underscores, e.g. "select_filter".
     const String command; /// Uppercased without last word, e.g. "SELECT".
+    const String description;
     const bool is_check;  /// E.g. false for SELECT_FILTER.
     static const RowPolicyFilterTypeInfo & get(RowPolicyFilterType type);
 };

src/Analyzer/ConstantNode.cpp

@@ -1,5 +1,7 @@
 #include <Analyzer/ConstantNode.h>
 
+#include <Analyzer/FunctionNode.h>
+
 #include <Common/assert_cast.h>
 #include <Common/FieldVisitorToString.h>
 #include <Common/SipHash.h>
@@ -38,52 +40,9 @@ ConstantNode::ConstantNode(Field value_)
     : ConstantNode(value_, applyVisitor(FieldToDataType(), value_))
 {}
 
-void ConstantNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const
+bool ConstantNode::requiresCastCall() const
-{
-    buffer << std::string(indent, ' ') << "CONSTANT id: " << format_state.getNodeId(this);
-
-    if (hasAlias())
-        buffer << ", alias: " << getAlias();
-
-    buffer << ", constant_value: " << constant_value->getValue().dump();
-    buffer << ", constant_value_type: " << constant_value->getType()->getName();
-
-    if (getSourceExpression())
-    {
-        buffer << '\n' << std::string(indent + 2, ' ') << "EXPRESSION" << '\n';
-        getSourceExpression()->dumpTreeImpl(buffer, format_state, indent + 4);
-    }
-}
-
-bool ConstantNode::isEqualImpl(const IQueryTreeNode & rhs) const
-{
-    const auto & rhs_typed = assert_cast<const ConstantNode &>(rhs);
-    return *constant_value == *rhs_typed.constant_value && value_string == rhs_typed.value_string;
-}
-
-void ConstantNode::updateTreeHashImpl(HashState & hash_state) const
-{
-    auto type_name = constant_value->getType()->getName();
-    hash_state.update(type_name.size());
-    hash_state.update(type_name);
-
-    hash_state.update(value_string.size());
-    hash_state.update(value_string);
-}
-
-QueryTreeNodePtr ConstantNode::cloneImpl() const
-{
-    return std::make_shared<ConstantNode>(constant_value, source_expression);
-}
-
-ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const
 {
     const auto & constant_value_literal = constant_value->getValue();
-    auto constant_value_ast = std::make_shared<ASTLiteral>(constant_value_literal);
-
-    if (!options.add_cast_for_constants)
-        return constant_value_ast;
-
     bool need_to_add_cast_function = false;
     auto constant_value_literal_type = constant_value_literal.getType();
     WhichDataType constant_value_type(constant_value->getType());
@@ -131,7 +90,72 @@ ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const
     // Add cast if constant was created as a result of constant folding.
     // Constant folding may lead to type transformation and literal on shard
     // may have a different type.
-    if (need_to_add_cast_function || source_expression != nullptr)
+    return need_to_add_cast_function || source_expression != nullptr;
+}
+
+bool ConstantNode::receivedFromInitiatorServer() const
+{
+    if (!hasSourceExpression())
+        return false;
+
+    auto * cast_function = getSourceExpression()->as<FunctionNode>();
+    if (!cast_function || cast_function->getFunctionName() != "_CAST")
+        return false;
+    return true;
+}
+
+void ConstantNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const
+{
+    buffer << std::string(indent, ' ') << "CONSTANT id: " << format_state.getNodeId(this);
+
+    if (hasAlias())
+        buffer << ", alias: " << getAlias();
+
+    buffer << ", constant_value: ";
+    if (mask_id)
+        buffer << "[HIDDEN id: " << mask_id << "]";
+    else
+        buffer << constant_value->getValue().dump();
+
+    buffer << ", constant_value_type: " << constant_value->getType()->getName();
+
+    if (!mask_id && getSourceExpression())
+    {
+        buffer << '\n' << std::string(indent + 2, ' ') << "EXPRESSION" << '\n';
+        getSourceExpression()->dumpTreeImpl(buffer, format_state, indent + 4);
+    }
+}
+
+bool ConstantNode::isEqualImpl(const IQueryTreeNode & rhs) const
+{
+    const auto & rhs_typed = assert_cast<const ConstantNode &>(rhs);
+    return *constant_value == *rhs_typed.constant_value && value_string == rhs_typed.value_string;
+}
+
+void ConstantNode::updateTreeHashImpl(HashState & hash_state) const
+{
+    auto type_name = constant_value->getType()->getName();
+    hash_state.update(type_name.size());
+    hash_state.update(type_name);
+
+    hash_state.update(value_string.size());
+    hash_state.update(value_string);
+}
+
+QueryTreeNodePtr ConstantNode::cloneImpl() const
+{
+    return std::make_shared<ConstantNode>(constant_value, source_expression);
+}
+
+ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const
+{
+    const auto & constant_value_literal = constant_value->getValue();
+    auto constant_value_ast = std::make_shared<ASTLiteral>(constant_value_literal);
+
+    if (!options.add_cast_for_constants)
+        return constant_value_ast;
+
+    if (requiresCastCall())
     {
         auto constant_type_name_ast = std::make_shared<ASTLiteral>(constant_value->getType()->getName());
         return makeASTFunction("_CAST", std::move(constant_value_ast), std::move(constant_type_name_ast));

src/Analyzer/ConstantNode.h

@@ -75,6 +75,17 @@ public:
         return constant_value->getType();
     }
 
+    /// Check if conversion to AST requires wrapping with _CAST function.
+    bool requiresCastCall() const;
+
+    /// Check if constant is a result of _CAST function constant folding.
+    bool receivedFromInitiatorServer() const;
+
+    void setMaskId(size_t id)
+    {
+        mask_id = id;
+    }
+
     void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override;
 
 protected:
@@ -90,6 +101,7 @@ private:
     ConstantValuePtr constant_value;
     String value_string;
     QueryTreeNodePtr source_expression;
+    size_t mask_id = 0;
 
     static constexpr size_t children_size = 0;
 };

src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h

@@ -0,0 +1,372 @@
+#pragma once
+
+#include <Parsers/FunctionSecretArgumentsFinder.h>
+#include <Analyzer/ConstantNode.h>
+#include <Analyzer/FunctionNode.h>
+#include <Analyzer/IQueryTreeNode.h>
+#include <Analyzer/IdentifierNode.h>
+#include <Analyzer/ListNode.h>
+#include <Common/KnownObjectNames.h>
+#include <Core/QualifiedTableName.h>
+
+#include <boost/algorithm/string/predicate.hpp>
+
+
+namespace DB
+{
+
+
+/// Finds arguments of a specified function which should not be displayed for most users for security reasons.
+/// That involves passwords and secret keys.
+class FunctionSecretArgumentsFinderTreeNode
+{
+public:
+    explicit FunctionSecretArgumentsFinderTreeNode(const FunctionNode & function_) : function(function_), arguments(function.getArguments())
+    {
+        if (arguments.getNodes().empty())
+            return;
+
+        findFunctionSecretArguments();
+    }
+
+    struct Result
+    {
+        /// Result constructed by default means no arguments will be hidden.
+        size_t start = static_cast<size_t>(-1);
+        size_t count = 0; /// Mostly it's either 0 or 1. There are only a few cases where `count` can be greater than 1 (e.g. see `encrypt`).
+                            /// In all known cases secret arguments are consecutive
+        bool are_named = false; /// Arguments like `password = 'password'` are considered as named arguments.
+        /// E.g. "headers" in `url('..', headers('foo' = '[HIDDEN]'))`
+        std::vector<std::string> nested_maps;
+
+        bool hasSecrets() const
+        {
+            return count != 0 || !nested_maps.empty();
+        }
+    };
+
+    FunctionSecretArgumentsFinder::Result getResult() const { return result; }
+
+private:
+    const FunctionNode & function;
+    const ListNode & arguments;
+    FunctionSecretArgumentsFinder::Result result;
+
+    void markSecretArgument(size_t index, bool argument_is_named = false)
+    {
+        if (index >= arguments.getNodes().size())
+            return;
+        if (!result.count)
+        {
+            result.start = index;
+            result.are_named = argument_is_named;
+        }
+        chassert(index >= result.start); /// We always check arguments consecutively
+        result.count = index + 1 - result.start;
+        if (!argument_is_named)
+            result.are_named = false;
+    }
+
+    void findFunctionSecretArguments()
+    {
+        const auto & name = function.getFunctionName();
+
+        if ((name == "mysql") || (name == "postgresql") || (name == "mongodb"))
+        {
+            /// mysql('host:port', 'database', 'table', 'user', 'password', ...)
+            /// postgresql('host:port', 'database', 'table', 'user', 'password', ...)
+            /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...)
+            findMySQLFunctionSecretArguments();
+        }
+        else if ((name == "s3") || (name == "cosn") || (name == "oss") ||
+                    (name == "deltaLake") || (name == "hudi") || (name == "iceberg"))
+        {
+            /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
+            findS3FunctionSecretArguments(/* is_cluster_function= */ false);
+        }
+        else if (name == "s3Cluster")
+        {
+            /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...)
+            findS3FunctionSecretArguments(/* is_cluster_function= */ true);
+        }
+        else if ((name == "remote") || (name == "remoteSecure"))
+        {
+            /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...)
+            findRemoteFunctionSecretArguments();
+        }
+        else if ((name == "encrypt") || (name == "decrypt") ||
+                    (name == "aes_encrypt_mysql") || (name == "aes_decrypt_mysql") ||
+                    (name == "tryDecrypt"))
+        {
+            /// encrypt('mode', 'plaintext', 'key' [, iv, aad])
+            findEncryptionFunctionSecretArguments();
+        }
+        else if (name == "url")
+        {
+            findURLSecretArguments();
+        }
+    }
+
+    void findMySQLFunctionSecretArguments()
+    {
+        if (isNamedCollectionName(0))
+        {
+            /// mysql(named_collection, ..., password = 'password', ...)
+            findSecretNamedArgument("password", 1);
+        }
+        else
+        {
+            /// mysql('host:port', 'database', 'table', 'user', 'password', ...)
+            markSecretArgument(4);
+        }
+    }
+
+    /// Returns the number of arguments excluding "headers" and "extra_credentials" (which should
+    /// always be at the end). Marks "headers" as secret, if found.
+    size_t excludeS3OrURLNestedMaps()
+    {
+        const auto & nodes = arguments.getNodes();
+        size_t count = nodes.size();
+        while (count > 0)
+        {
+            const FunctionNode * f = nodes.at(count - 1)->as<FunctionNode>();
+            if (!f)
+                break;
+            if (f->getFunctionName() == "headers")
+                result.nested_maps.push_back(f->getFunctionName());
+            else if (f->getFunctionName() != "extra_credentials")
+                break;
+            count -= 1;
+        }
+        return count;
+    }
+
+    void findS3FunctionSecretArguments(bool is_cluster_function)
+    {
+        /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument.
+        size_t url_arg_idx = is_cluster_function ? 1 : 0;
+
+        if (!is_cluster_function && isNamedCollectionName(0))
+        {
+            /// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...)
+            findSecretNamedArgument("secret_access_key", 1);
+            return;
+        }
+
+        /// We should check other arguments first because we don't need to do any replacement in case of
+        /// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
+        /// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
+        size_t count = excludeS3OrURLNestedMaps();
+        if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4))
+        {
+            String second_arg;
+            if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg))
+            {
+                if (boost::iequals(second_arg, "NOSIGN"))
+                    return; /// The argument after 'url' is "NOSIGN".
+
+                if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg))
+                    return; /// The argument after 'url' is a format: s3('url', 'format', ...)
+            }
+        }
+
+        /// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures:
+        /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
+        /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
+        if (url_arg_idx + 2 < count)
+            markSecretArgument(url_arg_idx + 2);
+    }
+
+    void findURLSecretArguments()
+    {
+        if (!isNamedCollectionName(0))
+            excludeS3OrURLNestedMaps();
+    }
+
+    bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const
+    {
+        if (arg_idx >= arguments.getNodes().size())
+            return false;
+
+        return tryGetStringFromArgument(arguments.getNodes()[arg_idx], res, allow_identifier);
+    }
+
+    static bool tryGetStringFromArgument(const QueryTreeNodePtr argument, String * res, bool allow_identifier = true)
+    {
+        if (const auto * literal = argument->as<ConstantNode>())
+        {
+            if (literal->getValue().getType() != Field::Types::String)
+                return false;
+            if (res)
+                *res = literal->getValue().safeGet<String>();
+            return true;
+        }
+
+        if (allow_identifier)
+        {
+            if (const auto * id = argument->as<IdentifierNode>())
+            {
+                if (res)
+                    *res = id->getIdentifier().getFullName();
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    void findRemoteFunctionSecretArguments()
+    {
+        if (isNamedCollectionName(0))
+        {
+            /// remote(named_collection, ..., password = 'password', ...)
+            findSecretNamedArgument("password", 1);
+            return;
+        }
+
+        /// We're going to replace 'password' with '[HIDDEN'] for the following signatures:
+        /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key])
+        /// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key])
+        /// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key])
+
+        /// But we should check the number of arguments first because we don't need to do any replacements in case of
+        /// remote('addresses_expr', db.table)
+        if (arguments.getNodes().size() < 3)
+            return;
+
+        size_t arg_num = 1;
+
+        /// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'.
+        const auto * table_function = arguments.getNodes()[arg_num]->as<FunctionNode>();
+        if (table_function && KnownTableFunctionNames::instance().exists(table_function->getFunctionName()))
+        {
+            ++arg_num;
+        }
+        else
+        {
+            std::optional<String> database;
+            std::optional<QualifiedTableName> qualified_table_name;
+            if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name))
+            {
+                /// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'.
+                /// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user'
+                /// before the argument 'password'. So it's safer to wipe two arguments just in case.
+                /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
+                /// before wiping it (because the `password` argument is always a literal string).
+                if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false))
+                {
+                    /// Wipe either `password` or `user`.
+                    markSecretArgument(arg_num + 2);
+                }
+                if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false))
+                {
+                    /// Wipe either `password` or `sharding_key`.
+                    markSecretArgument(arg_num + 3);
+                }
+                return;
+            }
+
+            /// Skip the current argument (which is either a database name or a qualified table name).
+            ++arg_num;
+            if (database)
+            {
+                /// Skip the 'table' argument if the previous argument was a database name.
+                ++arg_num;
+            }
+        }
+
+        /// Skip username.
+        ++arg_num;
+
+        /// Do our replacement:
+        /// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...)
+        /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
+        /// before wiping it (because the `password` argument is always a literal string).
+        bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false);
+        if (can_be_password)
+            markSecretArgument(arg_num);
+    }
+
+    /// Tries to get either a database name or a qualified table name from an argument.
+    /// Empty string is also allowed (it means the default database).
+    /// The function is used by findRemoteFunctionSecretArguments() to determine how many arguments to skip before a password.
+    bool tryGetDatabaseNameOrQualifiedTableName(
+        size_t arg_idx,
+        std::optional<String> & res_database,
+        std::optional<QualifiedTableName> & res_qualified_table_name) const
+    {
+        res_database.reset();
+        res_qualified_table_name.reset();
+
+        String str;
+        if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true))
+            return false;
+
+        if (str.empty())
+        {
+            res_database = "";
+            return true;
+        }
+
+        auto qualified_table_name = QualifiedTableName::tryParseFromString(str);
+        if (!qualified_table_name)
+            return false;
+
+        if (qualified_table_name->database.empty())
+            res_database = std::move(qualified_table_name->table);
+        else
+            res_qualified_table_name = std::move(qualified_table_name);
+        return true;
+    }
+
+    void findEncryptionFunctionSecretArguments()
+    {
+        if (arguments.getNodes().empty())
+            return;
+
+        /// We replace all arguments after 'mode' with '[HIDDEN]':
+        /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]')
+        result.start = 1;
+        result.count = arguments.getNodes().size() - 1;
+    }
+
+
+    /// Whether a specified argument can be the name of a named collection?
+    bool isNamedCollectionName(size_t arg_idx) const
+    {
+        if (arguments.getNodes().size() <= arg_idx)
+            return false;
+
+        const auto * identifier = arguments.getNodes()[arg_idx]->as<IdentifierNode>();
+        return identifier != nullptr;
+    }
+
+    /// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified.
+    void findSecretNamedArgument(const std::string_view & key, size_t start = 0)
+    {
+        for (size_t i = start; i < arguments.getNodes().size(); ++i)
+        {
+            const auto & argument = arguments.getNodes()[i];
+            const auto * equals_func = argument->as<FunctionNode>();
+            if (!equals_func || (equals_func->getFunctionName() != "equals"))
+                continue;
+
+            const auto * expr_list = equals_func->getArguments().as<ListNode>();
+            if (!expr_list)
+                continue;
+
+            const auto & equal_args = expr_list->getNodes();
+            if (equal_args.size() != 2)
+                continue;
+
+            String found_key;
+            if (!tryGetStringFromArgument(equal_args[0], &found_key))
+                continue;
+
+            if (found_key == key)
+                markSecretArgument(i, /* argument_is_named= */ true);
+        }
+    }
+};
+
+}

src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp

@@ -94,7 +94,8 @@ public:
         if (!func_node || func_node->getArguments().getNodes().size() != 1)
             return;
 
-        const auto * column_id = func_node->getArguments().getNodes()[0]->as<ColumnNode>();
+        const auto & argument_node = func_node->getArguments().getNodes()[0];
+        const auto * column_id = argument_node->as<ColumnNode>();
         if (!column_id)
             return;
 
@@ -119,7 +120,7 @@ public:
         if (!preimage_range)
             return;
 
-        const auto new_node = generateOptimizedDateFilter(comparator, *column_id, *preimage_range);
+        const auto new_node = generateOptimizedDateFilter(comparator, argument_node, *preimage_range);
 
         if (!new_node)
             return;
@@ -128,20 +129,22 @@ public:
     }
 
 private:
-    QueryTreeNodePtr
+    QueryTreeNodePtr generateOptimizedDateFilter(
-    generateOptimizedDateFilter(const String & comparator, const ColumnNode & column_node, const std::pair<Field, Field> & range) const
+        const String & comparator, const QueryTreeNodePtr & column_node, const std::pair<Field, Field> & range) const
     {
         const DateLUTImpl & date_lut = DateLUT::instance("UTC");
 
         String start_date_or_date_time;
         String end_date_or_date_time;
 
-        if (isDateOrDate32(column_node.getColumnType().get()))
+        const auto & column_node_typed = column_node->as<ColumnNode &>();
+        const auto & column_type = column_node_typed.getColumnType().get();
+        if (isDateOrDate32(column_type))
         {
             start_date_or_date_time = date_lut.dateToString(range.first.get<DateLUTImpl::Time>());
             end_date_or_date_time = date_lut.dateToString(range.second.get<DateLUTImpl::Time>());
         }
-        else if (isDateTime(column_node.getColumnType().get()) || isDateTime64(column_node.getColumnType().get()))
+        else if (isDateTime(column_type) || isDateTime64(column_type))
         {
             start_date_or_date_time = date_lut.timeToString(range.first.get<DateLUTImpl::Time>());
             end_date_or_date_time = date_lut.timeToString(range.second.get<DateLUTImpl::Time>());
@@ -151,69 +154,29 @@ private:
 
         if (comparator == "equals")
         {
-            const auto lhs = std::make_shared<FunctionNode>("greaterOrEquals");
+            return createFunctionNode(
-            lhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
+                "and",
-            lhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(start_date_or_date_time));
+                createFunctionNode("greaterOrEquals", column_node, std::make_shared<ConstantNode>(start_date_or_date_time)),
-            resolveOrdinaryFunctionNode(*lhs, lhs->getFunctionName());
+                createFunctionNode("less", column_node, std::make_shared<ConstantNode>(end_date_or_date_time)));
-
-            const auto rhs = std::make_shared<FunctionNode>("less");
-            rhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            rhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
-            resolveOrdinaryFunctionNode(*rhs, rhs->getFunctionName());
-
-            const auto new_date_filter = std::make_shared<FunctionNode>("and");
-            new_date_filter->getArguments().getNodes() = {lhs, rhs};
-            resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
-
-            return new_date_filter;
         }
         else if (comparator == "notEquals")
         {
-            const auto lhs = std::make_shared<FunctionNode>("less");
+            return createFunctionNode(
-            lhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
+                "or",
-            lhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(start_date_or_date_time));
+                createFunctionNode("less", column_node, std::make_shared<ConstantNode>(start_date_or_date_time)),
-            resolveOrdinaryFunctionNode(*lhs, lhs->getFunctionName());
+                createFunctionNode("greaterOrEquals", column_node, std::make_shared<ConstantNode>(end_date_or_date_time)));
-
-            const auto rhs = std::make_shared<FunctionNode>("greaterOrEquals");
-            rhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            rhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
-            resolveOrdinaryFunctionNode(*rhs, rhs->getFunctionName());
-
-            const auto new_date_filter = std::make_shared<FunctionNode>("or");
-            new_date_filter->getArguments().getNodes() = {lhs, rhs};
-            resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
-
-            return new_date_filter;
         }
         else if (comparator == "greater")
         {
-            const auto new_date_filter = std::make_shared<FunctionNode>("greaterOrEquals");
+            return createFunctionNode("greaterOrEquals", column_node, std::make_shared<ConstantNode>(end_date_or_date_time));
-            new_date_filter->getArguments().getNodes().push_back(
-                std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            new_date_filter->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
-            resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
-
-            return new_date_filter;
         }
         else if (comparator == "lessOrEquals")
         {
-            const auto new_date_filter = std::make_shared<FunctionNode>("less");
+            return createFunctionNode("less", column_node, std::make_shared<ConstantNode>(end_date_or_date_time));
-            new_date_filter->getArguments().getNodes().push_back(
-                std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            new_date_filter->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
-            resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
-
-            return new_date_filter;
         }
         else if (comparator == "less" || comparator == "greaterOrEquals")
         {
-            const auto new_date_filter = std::make_shared<FunctionNode>(comparator);
+            return createFunctionNode(comparator, column_node, std::make_shared<ConstantNode>(start_date_or_date_time));
-            new_date_filter->getArguments().getNodes().push_back(
-                std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            new_date_filter->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(start_date_or_date_time));
-            resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
-
-            return new_date_filter;
         }
         else [[unlikely]]
         {
@@ -224,10 +187,17 @@ private:
         }
     }
 
-    void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const
+    template <typename... Args>
+    QueryTreeNodePtr createFunctionNode(const String & function_name, Args &&... args) const
     {
         auto function = FunctionFactory::instance().get(function_name, getContext());
-        function_node.resolveAsFunction(function->build(function_node.getArgumentColumns()));
+        const auto function_node = std::make_shared<FunctionNode>(function_name);
+        auto & new_arguments = function_node->getArguments().getNodes();
+        new_arguments.reserve(sizeof...(args));
+        (new_arguments.push_back(std::forward<Args>(args)), ...);
+        function_node->resolveAsFunction(function->build(function_node->getArgumentColumns()));
+
+        return function_node;
     }
 };
 

src/Analyzer/Passes/QueryAnalysisPass.cpp

@@ -3,6 +3,7 @@
 #include <Common/checkStackSize.h>
 #include <Common/NamePrompter.h>
 #include <Common/ProfileEvents.h>
+#include <Analyzer/FunctionSecretArgumentsFinderTreeNode.h>
 
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
@@ -706,7 +707,10 @@ struct IdentifierResolveScope
         {
             subquery_depth = parent_scope->subquery_depth;
             context = parent_scope->context;
+            projection_mask_map = parent_scope->projection_mask_map;
         }
+        else
+            projection_mask_map = std::make_shared<std::map<IQueryTreeNode::Hash, size_t>>();
 
         if (auto * union_node = scope_node->as<UnionNode>())
         {
@@ -718,6 +722,11 @@ struct IdentifierResolveScope
             group_by_use_nulls = context->getSettingsRef().group_by_use_nulls &&
                 (query_node->isGroupByWithGroupingSets() || query_node->isGroupByWithRollup() || query_node->isGroupByWithCube());
         }
+
+        if (context)
+            join_use_nulls = context->getSettingsRef().join_use_nulls;
+        else if (parent_scope)
+            join_use_nulls = parent_scope->join_use_nulls;
     }
 
     QueryTreeNodePtr scope_node;
@@ -772,6 +781,8 @@ struct IdentifierResolveScope
 
     /// Apply nullability to aggregation keys
     bool group_by_use_nulls = false;
+    /// Join retutns NULLs instead of default values
+    bool join_use_nulls = false;
 
     /// JOINs count
     size_t joins_count = 0;
@@ -784,6 +795,9 @@ struct IdentifierResolveScope
       */
     QueryTreeNodePtr expression_join_tree_node;
 
+    /// Node hash to mask id map
+    std::shared_ptr<std::map<IQueryTreeNode::Hash, size_t>> projection_mask_map;
+
     [[maybe_unused]] const IdentifierResolveScope * getNearestQueryScope() const
     {
         const IdentifierResolveScope * scope_to_check = this;
@@ -1068,6 +1082,8 @@ private:
 class QueryAnalyzer
 {
 public:
+    explicit QueryAnalyzer(bool only_analyze_) : only_analyze(only_analyze_) {}
+
     void resolve(QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression, ContextPtr context)
     {
         IdentifierResolveScope scope(node, nullptr /*parent_scope*/);
@@ -1430,6 +1446,7 @@ private:
     /// Global scalar subquery to scalar value map
     std::unordered_map<QueryTreeNodePtrWithHash, Block> scalar_subquery_to_scalar_value;
 
+    const bool only_analyze;
 };
 
 /// Utility functions implementation
@@ -1977,11 +1994,26 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden
         auto interpreter = std::make_unique<InterpreterSelectQueryAnalyzer>(node->toAST(), subquery_context, subquery_context->getViewSource(), options);
 
         auto io = interpreter->execute();
-
         PullingAsyncPipelineExecutor executor(io.pipeline);
         io.pipeline.setProgressCallback(context->getProgressCallback());
         io.pipeline.setProcessListElement(context->getProcessListElement());
 
+        if (only_analyze)
+        {
+            /// If query is only analyzed, then constants are not correct.
+            scalar_block = interpreter->getSampleBlock();
+            for (auto & column : scalar_block)
+            {
+                if (column.column->empty())
+                {
+                    auto mut_col = column.column->cloneEmpty();
+                    mut_col->insertDefault();
+                    column.column = std::move(mut_col);
+                }
+            }
+        }
+        else
+        {
             Block block;
 
             while (block.rows() == 0 && executor.pull(block))
@@ -2053,6 +2085,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden
                         "tuple"});
                 }
             }
+        }
 
         scalar_subquery_to_scalar_value.emplace(node_with_hash, scalar_block);
     }
@@ -3286,7 +3319,6 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
     QueryTreeNodePtr resolved_identifier;
 
     JoinKind join_kind = from_join_node.getKind();
-    bool join_use_nulls = scope.context->getSettingsRef().join_use_nulls;
 
     /// If columns from left or right table were missed Object(Nullable('json')) subcolumns, they will be replaced
     /// to ConstantNode(NULL), which can't be cast to ColumnNode, so we resolve it here.
@@ -3451,7 +3483,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
     if (join_node_in_resolve_process || !resolved_identifier)
         return resolved_identifier;
 
-    if (join_use_nulls)
+    if (scope.join_use_nulls)
     {
         resolved_identifier = resolved_identifier->clone();
         convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side);
@@ -4439,7 +4471,7 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I
     else
         matched_expression_nodes_with_names = resolveUnqualifiedMatcher(matcher_node, scope);
 
-    if (scope.context->getSettingsRef().join_use_nulls)
+    if (scope.join_use_nulls)
     {
         /** If we are resolving matcher came from the result of JOIN and `join_use_nulls` is set,
           * we need to convert joined column type to Nullable.
@@ -5124,22 +5156,31 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
     }
 
     /// Resolve function arguments
-
     bool allow_table_expressions = is_special_function_in;
     auto arguments_projection_names = resolveExpressionNodeList(function_node_ptr->getArgumentsNode(),
         scope,
         true /*allow_lambda_expression*/,
         allow_table_expressions /*allow_table_expression*/);
 
-    if (function_node_ptr->toAST()->hasSecretParts())
+    /// Mask arguments if needed
+    if (!scope.context->getSettingsRef().format_display_secrets_in_show_and_select)
     {
-        for (auto & argument : arguments_projection_names)
+        if (FunctionSecretArgumentsFinder::Result secret_arguments = FunctionSecretArgumentsFinderTreeNode(*function_node_ptr).getResult(); secret_arguments.count)
         {
-            SipHash hash;
+            auto & argument_nodes = function_node_ptr->getArgumentsNode()->as<ListNode &>().getNodes();
-            hash.update(argument);
+
-            argument = getHexUIntLowercase(hash.get128());
+            for (size_t n = secret_arguments.start; n < secret_arguments.start + secret_arguments.count; ++n)
+            {
+                if (auto * constant = argument_nodes[n]->as<ConstantNode>())
+                {
+                    auto mask = scope.projection_mask_map->insert({constant->getTreeHash(), scope.projection_mask_map->size() + 1}).first->second;
+                    constant->setMaskId(mask);
+                    arguments_projection_names[n] = "[HIDDEN id: " + std::to_string(mask) + "]";
                 }
             }
+        }
+    }
+
     auto & function_node = *function_node_ptr;
 
     /// Replace right IN function argument if it is table or table function with subquery that read ordinary columns
@@ -7559,8 +7600,22 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
     }
 
     if (query_node_typed.getPrewhere())
+    {
+        /** Expression in PREWHERE with JOIN should not be modified by join_use_nulls.
+          * Example: SELECT * FROM t1 JOIN t2 USING (id) PREWHERE a = 1
+          * Column `a` should be resolved from table and should not change its type to Nullable.
+          */
+        bool join_use_nulls = scope.join_use_nulls;
+        bool use_identifier_lookup_to_result_cache = scope.use_identifier_lookup_to_result_cache;
+        scope.join_use_nulls = false;
+        scope.use_identifier_lookup_to_result_cache = false;
+
         resolveExpressionNode(query_node_typed.getPrewhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
 
+        scope.join_use_nulls = join_use_nulls;
+        scope.use_identifier_lookup_to_result_cache = use_identifier_lookup_to_result_cache;
+    }
+
     if (query_node_typed.getWhere())
         resolveExpressionNode(query_node_typed.getWhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
 
@@ -7750,13 +7805,16 @@ void QueryAnalyzer::resolveUnion(const QueryTreeNodePtr & union_node, Identifier
 
 }
 
-QueryAnalysisPass::QueryAnalysisPass(QueryTreeNodePtr table_expression_)
+QueryAnalysisPass::QueryAnalysisPass(QueryTreeNodePtr table_expression_, bool only_analyze_)
     : table_expression(std::move(table_expression_))
+    , only_analyze(only_analyze_)
 {}
 
+QueryAnalysisPass::QueryAnalysisPass(bool only_analyze_) : only_analyze(only_analyze_) {}
+
 void QueryAnalysisPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
 {
-    QueryAnalyzer analyzer;
+    QueryAnalyzer analyzer(only_analyze);
     analyzer.resolve(query_tree_node, table_expression, context);
     createUniqueTableAliases(query_tree_node, table_expression, context);
 }

src/Analyzer/Passes/QueryAnalysisPass.h

@@ -71,13 +71,13 @@ public:
     /** Construct query analysis pass for query or union analysis.
       * Available columns are extracted from query node join tree.
       */
-    QueryAnalysisPass() = default;
+    explicit QueryAnalysisPass(bool only_analyze_ = false);
 
     /** Construct query analysis pass for expression or list of expressions analysis.
       * Available expression columns are extracted from table expression.
       * Table expression node must have query, union, table, table function type.
       */
-    explicit QueryAnalysisPass(QueryTreeNodePtr table_expression_);
+    QueryAnalysisPass(QueryTreeNodePtr table_expression_, bool only_analyze_ = false);
 
     String getName() override
     {
@@ -93,6 +93,7 @@ public:
 
 private:
     QueryTreeNodePtr table_expression;
+    const bool only_analyze;
 };
 
 }

src/Analyzer/QueryTreePassManager.cpp

@@ -246,9 +246,9 @@ void QueryTreePassManager::dump(WriteBuffer & buffer, size_t up_to_pass_index)
     }
 }
 
-void addQueryTreePasses(QueryTreePassManager & manager)
+void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze)
 {
-    manager.addPass(std::make_unique<QueryAnalysisPass>());
+    manager.addPass(std::make_unique<QueryAnalysisPass>(only_analyze));
     manager.addPass(std::make_unique<GroupingFunctionsResolvePass>());
 
     manager.addPass(std::make_unique<RemoveUnusedProjectionColumnsPass>());

src/Analyzer/QueryTreePassManager.h

@@ -47,6 +47,6 @@ private:
     std::vector<QueryTreePassPtr> passes;
 };
 
-void addQueryTreePasses(QueryTreePassManager & manager);
+void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze = false);
 
 }

src/Backups/BackupsWorker.cpp

@@ -18,6 +18,7 @@
 #include <Interpreters/executeDDLQueryOnCluster.h>
 #include <Parsers/ASTBackupQuery.h>
 #include <Parsers/ASTFunction.h>
+#include <Common/CurrentThread.h>
 #include <Common/Exception.h>
 #include <Common/Macros.h>
 #include <Common/logger_useful.h>
@@ -486,7 +487,7 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context
             /// process_list_element_holder is used to make an element in ProcessList live while BACKUP is working asynchronously.
             auto process_list_element = context_in_use->getProcessListElement();
 
-            scheduleFromThreadPool<void>(
+            thread_pool.scheduleOrThrowOnError(
                 [this,
                  backup_query,
                  backup_id,
@@ -502,6 +503,8 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context
                     BackupMutablePtr backup_async;
                     try
                     {
+                        setThreadName("BackupWorker");
+                        CurrentThread::QueryScope query_scope(context_in_use);
                         doBackup(
                             backup_async,
                             backup_query,
@@ -517,8 +520,7 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context
                     {
                         on_exception(backup_async, backup_id, backup_name_for_logging, backup_settings, backup_coordination);
                     }
-                },
+                });
-                thread_pool, "BackupWorker");
         }
         else
         {
@@ -864,7 +866,7 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt
             /// process_list_element_holder is used to make an element in ProcessList live while RESTORE is working asynchronously.
             auto process_list_element = context_in_use->getProcessListElement();
 
-            scheduleFromThreadPool<void>(
+            thread_pool.scheduleOrThrowOnError(
                 [this,
                  restore_query,
                  restore_id,
@@ -878,6 +880,8 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt
                 {
                     try
                     {
+                        setThreadName("RestorerWorker");
+                        CurrentThread::QueryScope query_scope(context_in_use);
                         doRestore(
                             restore_query,
                             restore_id,
@@ -891,9 +895,7 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt
                     {
                         on_exception(restore_id, backup_name_for_logging, restore_settings, restore_coordination);
                     }
-                },
+                });
-                thread_pool,
-                "RestoreWorker");
         }
         else
         {

src/Client/ConnectionParameters.cpp

@@ -115,7 +115,7 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
     /// At the same time, I want clickhouse-local to always work, regardless.
     /// TODO: get rid of glibc, or replace getaddrinfo to c-ares.
 
-    compression = config.getBool("compression", host != "localhost" && !isLocalAddress(DNSResolver::instance().resolveHost(host)))
+    compression = config.getBool("compression", host != "localhost" && !isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(host).front()))
                   ? Protocol::Compression::Enable : Protocol::Compression::Disable;
 
     timeouts = ConnectionTimeouts()

src/Columns/ColumnUnique.cpp

@@ -0,0 +1,25 @@
+#include <Columns/ColumnUnique.h>
+
+namespace DB
+{
+
+/// Explicit template instantiations.
+template class ColumnUnique<ColumnInt8>;
+template class ColumnUnique<ColumnUInt8>;
+template class ColumnUnique<ColumnInt16>;
+template class ColumnUnique<ColumnUInt16>;
+template class ColumnUnique<ColumnInt32>;
+template class ColumnUnique<ColumnUInt32>;
+template class ColumnUnique<ColumnInt64>;
+template class ColumnUnique<ColumnUInt64>;
+template class ColumnUnique<ColumnInt128>;
+template class ColumnUnique<ColumnUInt128>;
+template class ColumnUnique<ColumnInt256>;
+template class ColumnUnique<ColumnUInt256>;
+template class ColumnUnique<ColumnFloat32>;
+template class ColumnUnique<ColumnFloat64>;
+template class ColumnUnique<ColumnString>;
+template class ColumnUnique<ColumnFixedString>;
+template class ColumnUnique<ColumnDateTime64>;
+
+}

src/Columns/ColumnUnique.h

@@ -15,6 +15,8 @@
 #include <Common/typeid_cast.h>
 #include <Common/assert_cast.h>
 #include <Common/FieldVisitors.h>
+#include "Columns/ColumnsDateTime.h"
+#include "Columns/ColumnsNumber.h"
 
 #include <base/range.h>
 #include <base/unaligned.h>
@@ -736,4 +738,23 @@ UInt128 ColumnUnique<ColumnType>::IncrementalHash::getHash(const ColumnType & co
     return cur_hash;
 }
 
+
+extern template class ColumnUnique<ColumnInt8>;
+extern template class ColumnUnique<ColumnUInt8>;
+extern template class ColumnUnique<ColumnInt16>;
+extern template class ColumnUnique<ColumnUInt16>;
+extern template class ColumnUnique<ColumnInt32>;
+extern template class ColumnUnique<ColumnUInt32>;
+extern template class ColumnUnique<ColumnInt64>;
+extern template class ColumnUnique<ColumnUInt64>;
+extern template class ColumnUnique<ColumnInt128>;
+extern template class ColumnUnique<ColumnUInt128>;
+extern template class ColumnUnique<ColumnInt256>;
+extern template class ColumnUnique<ColumnUInt256>;
+extern template class ColumnUnique<ColumnFloat32>;
+extern template class ColumnUnique<ColumnFloat64>;
+extern template class ColumnUnique<ColumnString>;
+extern template class ColumnUnique<ColumnFixedString>;
+extern template class ColumnUnique<ColumnDateTime64>;
+
 }

src/Common/DNSResolver.cpp

@@ -202,10 +202,10 @@ DNSResolver::DNSResolver() : impl(std::make_unique<DNSResolver::Impl>()), log(ge
 
 Poco::Net::IPAddress DNSResolver::resolveHost(const std::string & host)
 {
-    return pickAddress(resolveHostAll(host));
+    return pickAddress(resolveHostAll(host)); // random order -> random pick
 }
 
-DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
+DNSResolver::IPAddresses DNSResolver::resolveHostAllInOriginOrder(const std::string & host)
 {
     if (impl->disable_cache)
         return resolveIPAddressImpl(host);
@@ -214,6 +214,13 @@ DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
     return resolveIPAddressWithCache(impl->cache_host, host);
 }
 
+DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
+{
+    auto addresses = resolveHostAllInOriginOrder(host);
+    std::shuffle(addresses.begin(), addresses.end(), thread_local_rng);
+    return addresses;
+}
+
 Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host_and_port)
 {
     if (impl->disable_cache)

src/Common/DNSResolver.h

@@ -34,6 +34,9 @@ public:
     Poco::Net::IPAddress resolveHost(const std::string & host);
 
     /// Accepts host names like 'example.com' or '127.0.0.1' or '::1' and resolves all its IPs
+    /// resolveHostAllInOriginOrder returns addresses with the same order as system call returns it
+    IPAddresses resolveHostAllInOriginOrder(const std::string & host);
+    /// resolveHostAll returns addresses in random order
     IPAddresses resolveHostAll(const std::string & host);
 
     /// Accepts host names like 'example.com:port' or '127.0.0.1:port' or '[::1]:port' and resolves its IP and port

src/Common/DateLUTImpl.h

@@ -3,13 +3,13 @@
 #include <base/DayNum.h>
 #include <base/defines.h>
 #include <base/types.h>
+#include <Core/DecimalFunctions.h>
 
 #include <ctime>
 #include <cassert>
 #include <string>
 #include <type_traits>
 
-
 #define DATE_SECONDS_PER_DAY 86400 /// Number of seconds in a day, 60 * 60 * 24
 
 #define DATE_LUT_MIN_YEAR 1900 /// 1900 since majority of financial organizations consider 1900 as an initial year.
@@ -280,9 +280,9 @@ private:
         static_assert(std::is_integral_v<DateOrTime> && std::is_integral_v<Divisor>);
         assert(divisor > 0);
 
-        if (likely(offset_is_whole_number_of_hours_during_epoch))
+        if (offset_is_whole_number_of_hours_during_epoch) [[likely]]
         {
-            if (likely(x >= 0))
+            if (x >= 0) [[likely]]
                 return static_cast<DateOrTime>(x / divisor * divisor);
 
             /// Integer division for negative numbers rounds them towards zero (up).
@@ -576,10 +576,10 @@ public:
 
     unsigned toSecond(Time t) const
     {
-        if (likely(offset_is_whole_number_of_minutes_during_epoch))
+        if (offset_is_whole_number_of_minutes_during_epoch) [[likely]]
         {
             Time res = t % 60;
-            if (likely(res >= 0))
+            if (res >= 0) [[likely]]
                 return static_cast<unsigned>(res);
             return static_cast<unsigned>(res) + 60;
         }
@@ -593,6 +593,30 @@ public:
         return time % 60;
     }
 
+    template <typename DateOrTime>
+    unsigned toMillisecond(const DateOrTime & datetime, Int64 scale_multiplier) const
+    {
+        constexpr Int64 millisecond_multiplier = 1'000;
+        constexpr Int64 microsecond_multiplier = 1'000 * millisecond_multiplier;
+        constexpr Int64 divider = microsecond_multiplier / millisecond_multiplier;
+
+        auto components = DB::DecimalUtils::splitWithScaleMultiplier(datetime, scale_multiplier);
+
+        if (datetime.value < 0 && components.fractional)
+        {
+            components.fractional = scale_multiplier + (components.whole ? Int64(-1) : Int64(1)) * components.fractional;
+            --components.whole;
+        }
+        Int64 fractional = components.fractional;
+        if (scale_multiplier > microsecond_multiplier)
+            fractional = fractional / (scale_multiplier / microsecond_multiplier);
+        else if (scale_multiplier < microsecond_multiplier)
+            fractional = fractional * (microsecond_multiplier / scale_multiplier);
+
+        UInt16 millisecond = static_cast<UInt16>(fractional / divider);
+        return millisecond;
+    }
+
     unsigned toMinute(Time t) const
     {
         if (t >= 0 && offset_is_whole_number_of_hours_during_epoch)
@@ -1122,9 +1146,9 @@ public:
     DateOrTime toStartOfMinuteInterval(DateOrTime t, UInt64 minutes) const
     {
         Int64 divisor = 60 * minutes;
-        if (likely(offset_is_whole_number_of_minutes_during_epoch))
+        if (offset_is_whole_number_of_minutes_during_epoch) [[likely]]
         {
-            if (likely(t >= 0))
+            if (t >= 0) [[likely]]
                 return static_cast<DateOrTime>(t / divisor * divisor);
             return static_cast<DateOrTime>((t + 1 - divisor) / divisor * divisor);
         }
@@ -1339,7 +1363,7 @@ public:
 
     UInt8 saturateDayOfMonth(Int16 year, UInt8 month, UInt8 day_of_month) const
     {
-        if (likely(day_of_month <= 28))
+        if (day_of_month <= 28) [[likely]]
             return day_of_month;
 
         UInt8 days_in_month = daysInMonth(year, month);

src/Common/FieldVisitorConvertToNumber.cpp

@@ -0,0 +1,23 @@
+#include <Common/FieldVisitorConvertToNumber.h>
+#include "base/Decimal.h"
+
+namespace DB
+{
+
+/// Explicit template instantiations.
+template class FieldVisitorConvertToNumber<Int8>;
+template class FieldVisitorConvertToNumber<UInt8>;
+template class FieldVisitorConvertToNumber<Int16>;
+template class FieldVisitorConvertToNumber<UInt16>;
+template class FieldVisitorConvertToNumber<Int32>;
+template class FieldVisitorConvertToNumber<UInt32>;
+template class FieldVisitorConvertToNumber<Int64>;
+template class FieldVisitorConvertToNumber<UInt64>;
+template class FieldVisitorConvertToNumber<Int128>;
+template class FieldVisitorConvertToNumber<UInt128>;
+template class FieldVisitorConvertToNumber<Int256>;
+template class FieldVisitorConvertToNumber<UInt256>;
+template class FieldVisitorConvertToNumber<Float32>;
+template class FieldVisitorConvertToNumber<Float64>;
+
+}

src/Common/FieldVisitorConvertToNumber.h

@@ -117,4 +117,19 @@ public:
     T operator() (const bool & x) const { return T(x); }
 };
 
+extern template class FieldVisitorConvertToNumber<Int8>;
+extern template class FieldVisitorConvertToNumber<UInt8>;
+extern template class FieldVisitorConvertToNumber<Int16>;
+extern template class FieldVisitorConvertToNumber<UInt16>;
+extern template class FieldVisitorConvertToNumber<Int32>;
+extern template class FieldVisitorConvertToNumber<UInt32>;
+extern template class FieldVisitorConvertToNumber<Int64>;
+extern template class FieldVisitorConvertToNumber<UInt64>;
+extern template class FieldVisitorConvertToNumber<Int128>;
+extern template class FieldVisitorConvertToNumber<UInt128>;
+extern template class FieldVisitorConvertToNumber<Int256>;
+extern template class FieldVisitorConvertToNumber<UInt256>;
+extern template class FieldVisitorConvertToNumber<Float32>;
+extern template class FieldVisitorConvertToNumber<Float64>;
+
 }

src/Common/ProfileEvents.cpp

@@ -475,6 +475,7 @@ The server successfully detected this situation and will download merged part fr
     M(FileSegmentUseMicroseconds, "File segment use() time") \
     M(FileSegmentRemoveMicroseconds, "File segment remove() time") \
     M(FileSegmentHolderCompleteMicroseconds, "File segments holder complete() time") \
+    M(FileSegmentFailToIncreasePriority, "Number of times the priority was not increased due to a high contention on the cache lock") \
     M(FilesystemCacheHoldFileSegments, "Filesystem cache file segments count, which were hold") \
     M(FilesystemCacheUnusedHoldFileSegments, "Filesystem cache file segments count, which were hold, but not used (because of seek or LIMIT n, etc)") \
     \

src/Coordination/KeeperStateManager.cpp

@@ -30,7 +30,7 @@ bool isLocalhost(const std::string & hostname)
 {
     try
     {
-        return isLocalAddress(DNSResolver::instance().resolveHost(hostname));
+        return isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(hostname).front());
     }
     catch (...)
     {

src/Core/Settings.h

@@ -186,6 +186,7 @@ class IColumn;
     \
     M(String, cluster_for_parallel_replicas, "", "Cluster for a shard in which current server is located", 0) \
     M(UInt64, allow_experimental_parallel_reading_from_replicas, 0, "Use all the replicas from a shard for SELECT query execution. Reading is parallelized and coordinated dynamically. 0 - disabled, 1 - enabled, silently disable them in case of failure, 2 - enabled, throw an exception in case of failure", 0) \
+    M(Bool, parallel_replicas_allow_in_with_subquery, true, "If true, subquery for IN will be executed on every follower replica.", 0) \
     M(Float, parallel_replicas_single_task_marks_count_multiplier, 2, "A multiplier which will be added during calculation for minimal number of marks to retrieve from coordinator. This will be applied only for remote replicas.", 0) \
     M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \
     M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \
@@ -597,6 +598,7 @@ class IColumn;
     M(Bool, normalize_function_names, true, "Normalize function names to their canonical names", 0) \
     M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there are constants there", 0) \
     M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views if the block is not a duplicate for the table. Use true to always deduplicate in dependent tables.", 0) \
+    M(Bool, throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert, true, "Throw exception on INSERT query when the setting `deduplicate_blocks_in_dependent_materialized_views` is enabled along with `async_insert`. It guarantees correctness, because these features can't work together.", 0) \
     M(Bool, update_insert_deduplication_token_in_dependent_materialized_views, false, "Should update insert deduplication token with table identifier during insert in dependent materialized views.", 0) \
     M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \
     M(Bool, ignore_materialized_views_with_dropped_target_table, false, "Ignore MVs with dropped target table during pushing to views", 0) \
@@ -697,7 +699,7 @@ class IColumn;
     M(Bool, database_replicated_allow_replicated_engine_arguments, true, "Allow to create only Replicated tables in database with engine Replicated with explicit arguments", 0) \
     M(Bool, cloud_mode, false, "Only available in ClickHouse Cloud", 0) \
     M(UInt64, cloud_mode_engine, 1, "Only available in ClickHouse Cloud", 0) \
-    M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result, one of: 'none', 'throw', 'null_status_on_timeout', 'never_throw'", 0) \
+    M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result, one of: 'none', 'throw', 'null_status_on_timeout', 'never_throw', 'none_only_active', 'throw_only_active', 'null_status_on_timeout_only_active'", 0) \
     M(UInt64, distributed_ddl_entry_format_version, 5, "Compatibility version of distributed DDL (ON CLUSTER) queries", 0) \
     \
     M(UInt64, external_storage_max_read_rows, 0, "Limit maximum number of rows when table with external engine should flush history data. Now supported only for MySQL table engine, database engine, dictionary and MaterializedMySQL. If equal to 0, this setting is disabled", 0) \

src/Core/SettingsChangesHistory.h

@@ -90,6 +90,8 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
               {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
               {"page_cache_inject_eviction", false, false, "Added userspace page cache"},
               {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
+              {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."},
+              {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
               }},
     {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
               {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},

src/Core/SettingsEnums.cpp

@@ -118,6 +118,7 @@ IMPLEMENT_SETTING_ENUM(DistributedDDLOutputMode, ErrorCodes::BAD_ARGUMENTS,
      {"null_status_on_timeout", DistributedDDLOutputMode::NULL_STATUS_ON_TIMEOUT},
      {"throw_only_active", DistributedDDLOutputMode::THROW_ONLY_ACTIVE},
      {"null_status_on_timeout_only_active", DistributedDDLOutputMode::NULL_STATUS_ON_TIMEOUT_ONLY_ACTIVE},
+     {"none_only_active", DistributedDDLOutputMode::NONE_ONLY_ACTIVE},
      {"never_throw", DistributedDDLOutputMode::NEVER_THROW}})
 
 IMPLEMENT_SETTING_ENUM(StreamingHandleErrorMode, ErrorCodes::BAD_ARGUMENTS,

src/Core/SettingsEnums.h

@@ -177,6 +177,7 @@ enum class DistributedDDLOutputMode
     NEVER_THROW,
     THROW_ONLY_ACTIVE,
     NULL_STATUS_ON_TIMEOUT_ONLY_ACTIVE,
+    NONE_ONLY_ACTIVE,
 };
 
 DECLARE_SETTING_ENUM(DistributedDDLOutputMode)

src/DataTypes/DataTypeDecimalBase.h

@@ -207,4 +207,10 @@ inline DataTypePtr createDecimal(UInt64 precision_value, UInt64 scale_value)
     return std::make_shared<DecimalType<Decimal256>>(precision_value, scale_value);
 }
 
+extern template class DataTypeDecimalBase<Decimal32>;
+extern template class DataTypeDecimalBase<Decimal64>;
+extern template class DataTypeDecimalBase<Decimal128>;
+extern template class DataTypeDecimalBase<Decimal256>;
+extern template class DataTypeDecimalBase<DateTime64>;
+
 }

src/DataTypes/DataTypesDecimal.cpp

@@ -112,6 +112,256 @@ static DataTypePtr createExact(const ASTPtr & arguments)
     return createDecimal<DataTypeDecimal>(precision, scale);
 }
 
+template <typename FromDataType, typename ToDataType, typename ReturnType>
+requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
+ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
+{
+    using FromFieldType = typename FromDataType::FieldType;
+    using ToFieldType = typename ToDataType::FieldType;
+    using MaxFieldType = std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), FromFieldType, ToFieldType>;
+    using MaxNativeType = typename MaxFieldType::NativeType;
+
+    static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
+
+    MaxNativeType converted_value;
+    if (scale_to > scale_from)
+    {
+        converted_value = DecimalUtils::scaleMultiplier<MaxNativeType>(scale_to - scale_from);
+        if (common::mulOverflow(static_cast<MaxNativeType>(value.value), converted_value, converted_value))
+        {
+            if constexpr (throw_exception)
+                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow while multiplying {} by scale {}",
+                                std::string(ToDataType::family_name), toString(value.value), toString(converted_value));
+            else
+                return ReturnType(false);
+        }
+    }
+    else if (scale_to == scale_from)
+    {
+        converted_value = value.value;
+    }
+    else
+    {
+        converted_value = value.value / DecimalUtils::scaleMultiplier<MaxNativeType>(scale_from - scale_to);
+    }
+
+    if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType))
+    {
+        if (converted_value < std::numeric_limits<typename ToFieldType::NativeType>::min() ||
+            converted_value > std::numeric_limits<typename ToFieldType::NativeType>::max())
+        {
+            if constexpr (throw_exception)
+                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow: {} is not in range ({}, {})",
+                                std::string(ToDataType::family_name), toString(converted_value),
+                                toString(std::numeric_limits<typename ToFieldType::NativeType>::min()),
+                                toString(std::numeric_limits<typename ToFieldType::NativeType>::max()));
+            else
+                return ReturnType(false);
+        }
+    }
+
+    result = static_cast<typename ToFieldType::NativeType>(converted_value);
+
+    return ReturnType(true);
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template void convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, void>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result); \
+    template bool convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, bool>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
+typename ToDataType::FieldType convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to)
+{
+    using ToFieldType = typename ToDataType::FieldType;
+    ToFieldType result;
+
+    convertDecimalsImpl<FromDataType, ToDataType, void>(value, scale_from, scale_to, result);
+
+    return result;
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template typename TO_DATA_TYPE::FieldType convertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
+bool tryConvertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
+{
+    return convertDecimalsImpl<FromDataType, ToDataType, bool>(value, scale_from, scale_to, result);
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template bool tryConvertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType, typename ReturnType>
+requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
+ReturnType convertFromDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType & result)
+{
+    using FromFieldType = typename FromDataType::FieldType;
+    using ToFieldType = typename ToDataType::FieldType;
+
+    return DecimalUtils::convertToImpl<ToFieldType, FromFieldType, ReturnType>(value, scale, result);
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template void convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result); \
+    template bool convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
+inline typename ToDataType::FieldType convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
+{
+    typename ToDataType::FieldType result;
+
+    convertFromDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
+
+    return result;
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template typename TO_DATA_TYPE::FieldType convertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
+inline bool tryConvertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
+{
+    return convertFromDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template bool tryConvertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType, typename ReturnType>
+requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
+ReturnType convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType & result)
+{
+    using FromFieldType = typename FromDataType::FieldType;
+    using ToFieldType = typename ToDataType::FieldType;
+    using ToNativeType = typename ToFieldType::NativeType;
+
+    static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
+
+    if constexpr (std::is_floating_point_v<FromFieldType>)
+    {
+        if (!std::isfinite(value))
+        {
+            if constexpr (throw_exception)
+                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Cannot convert infinity or NaN to decimal", ToDataType::family_name);
+            else
+                return ReturnType(false);
+        }
+
+        auto out = value * static_cast<FromFieldType>(DecimalUtils::scaleMultiplier<ToNativeType>(scale));
+
+        if (out <= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::min()) ||
+            out >= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::max()))
+        {
+            if constexpr (throw_exception)
+                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Float is out of Decimal range", ToDataType::family_name);
+            else
+                return ReturnType(false);
+        }
+
+        result = static_cast<ToNativeType>(out);
+        return ReturnType(true);
+    }
+    else
+    {
+        if constexpr (is_big_int_v<FromFieldType>)
+            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal256>, ToDataType, ReturnType>(static_cast<Int256>(value), 0, scale, result));
+        else if constexpr (std::is_same_v<FromFieldType, UInt64>)
+            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal128>, ToDataType, ReturnType>(static_cast<Int128>(value), 0, scale, result));
+        else
+            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal64>, ToDataType, ReturnType>(static_cast<Int64>(value), 0, scale, result));
+    }
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template void convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);  \
+    template bool convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
+inline typename ToDataType::FieldType convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
+{
+    typename ToDataType::FieldType result;
+    convertToDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
+    return result;
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template typename TO_DATA_TYPE::FieldType convertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
+inline bool tryConvertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
+{
+    return convertToDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template bool tryConvertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename T>
+DataTypePtr createDecimalMaxPrecision(UInt64 scale)
+{
+    return std::make_shared<DataTypeDecimal<T>>(DecimalUtils::max_precision<T>, scale);
+}
+
+template DataTypePtr createDecimalMaxPrecision<Decimal32>(UInt64 scale);
+template DataTypePtr createDecimalMaxPrecision<Decimal64>(UInt64 scale);
+template DataTypePtr createDecimalMaxPrecision<Decimal128>(UInt64 scale);
+template DataTypePtr createDecimalMaxPrecision<Decimal256>(UInt64 scale);
+
+/// Explicit template instantiations.
+template class DataTypeDecimal<Decimal32>;
+template class DataTypeDecimal<Decimal64>;
+template class DataTypeDecimal<Decimal128>;
+template class DataTypeDecimal<Decimal256>;
+
 void registerDataTypeDecimal(DataTypeFactory & factory)
 {
     factory.registerDataType("Decimal32", createExact<Decimal32>, DataTypeFactory::CaseInsensitive);
@@ -125,10 +375,4 @@ void registerDataTypeDecimal(DataTypeFactory & factory)
     factory.registerAlias("FIXED", "Decimal", DataTypeFactory::CaseInsensitive);
 }
 
-/// Explicit template instantiations.
-template class DataTypeDecimal<Decimal32>;
-template class DataTypeDecimal<Decimal64>;
-template class DataTypeDecimal<Decimal128>;
-template class DataTypeDecimal<Decimal256>;
-
 }

src/DataTypes/DataTypesDecimal.h

@@ -3,7 +3,11 @@
 #include <base/arithmeticOverflow.h>
 #include <base/extended_types.h>
 #include <Common/typeid_cast.h>
+#include <base/Decimal.h>
+#include <base/Decimal_fwd.h>
 #include <DataTypes/IDataType.h>
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDecimalBase.h>
 #include <DataTypes/DataTypeDateTime64.h>
 
@@ -13,7 +17,6 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int DECIMAL_OVERFLOW;
     extern const int LOGICAL_ERROR;
 }
 
@@ -99,171 +102,145 @@ inline UInt32 getDecimalScale(const DataTypeDecimal<T> & data_type)
     return data_type.getScale();
 }
 
+#define FOR_EACH_DECIMAL_TYPE(M) \
+    M(DataTypeDecimal<DateTime64>) \
+    M(DataTypeDateTime64) \
+    M(DataTypeDecimal32) \
+    M(DataTypeDecimal64) \
+    M(DataTypeDecimal128) \
+    M(DataTypeDecimal256)
+
+#define FOR_EACH_DECIMAL_TYPE_PASS(M, X) \
+    M(DataTypeDecimal<DateTime64>, X) \
+    M(DataTypeDateTime64, X) \
+    M(DataTypeDecimal32, X) \
+    M(DataTypeDecimal64, X) \
+    M(DataTypeDecimal128, X) \
+    M(DataTypeDecimal256, X)
+
+
 template <typename FromDataType, typename ToDataType, typename ReturnType = void>
 requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
-inline ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
+ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result);
-{
-    using FromFieldType = typename FromDataType::FieldType;
-    using ToFieldType = typename ToDataType::FieldType;
-    using MaxFieldType = std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), FromFieldType, ToFieldType>;
-    using MaxNativeType = typename MaxFieldType::NativeType;
 
-    static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template void convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, void>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result); \
+    extern template bool convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, bool>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
 
-    MaxNativeType converted_value;
-    if (scale_to > scale_from)
-    {
-        converted_value = DecimalUtils::scaleMultiplier<MaxNativeType>(scale_to - scale_from);
-        if (common::mulOverflow(static_cast<MaxNativeType>(value.value), converted_value, converted_value))
-        {
-            if constexpr (throw_exception)
-                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow while multiplying {} by scale {}",
-                                std::string(ToDataType::family_name), toString(value.value), toString(converted_value));
-            else
-                return ReturnType(false);
-        }
-    }
-    else if (scale_to == scale_from)
-    {
-        converted_value = value.value;
-    }
-    else
-    {
-        converted_value = value.value / DecimalUtils::scaleMultiplier<MaxNativeType>(scale_from - scale_to);
-    }
-
-    if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType))
-    {
-        if (converted_value < std::numeric_limits<typename ToFieldType::NativeType>::min() ||
-            converted_value > std::numeric_limits<typename ToFieldType::NativeType>::max())
-        {
-            if constexpr (throw_exception)
-                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow: {} is not in range ({}, {})",
-                                std::string(ToDataType::family_name), toString(converted_value),
-                                toString(std::numeric_limits<typename ToFieldType::NativeType>::min()),
-                                toString(std::numeric_limits<typename ToFieldType::NativeType>::max()));
-            else
-                return ReturnType(false);
-        }
-    }
-
-    result = static_cast<typename ToFieldType::NativeType>(converted_value);
-
-    return ReturnType(true);
-}
 
 template <typename FromDataType, typename ToDataType>
 requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
-inline typename ToDataType::FieldType convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to)
+typename ToDataType::FieldType convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to);
-{
-    using ToFieldType = typename ToDataType::FieldType;
-    ToFieldType result;
 
-    convertDecimalsImpl<FromDataType, ToDataType, void>(value, scale_from, scale_to, result);
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template typename TO_DATA_TYPE::FieldType convertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
 
-    return result;
-}
 
 template <typename FromDataType, typename ToDataType>
 requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
-inline bool tryConvertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
+bool tryConvertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result);
-{
+
-    return convertDecimalsImpl<FromDataType, ToDataType, bool>(value, scale_from, scale_to, result);
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
-}
+    extern template bool tryConvertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
 
 template <typename FromDataType, typename ToDataType, typename ReturnType>
 requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
-inline ReturnType convertFromDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
+ReturnType convertFromDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType & result);
-{
+
-    using FromFieldType = typename FromDataType::FieldType;
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
-    using ToFieldType = typename ToDataType::FieldType;
+    extern template void convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result); \
+    extern template bool convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
 
-    return DecimalUtils::convertToImpl<ToFieldType, FromFieldType, ReturnType>(value, scale, result);
-}
 
 template <typename FromDataType, typename ToDataType>
 requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
-inline typename ToDataType::FieldType convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
+typename ToDataType::FieldType convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale);
-{
-    typename ToDataType::FieldType result;
 
-    convertFromDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template typename TO_DATA_TYPE::FieldType convertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
 
-    return result;
-}
 
 template <typename FromDataType, typename ToDataType>
 requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
-inline bool tryConvertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
+bool tryConvertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result);
-{
+
-    return convertFromDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
-}
+    extern template bool tryConvertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
 
 template <typename FromDataType, typename ToDataType, typename ReturnType>
 requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
-inline ReturnType convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
+ReturnType convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result);
-{
-    using FromFieldType = typename FromDataType::FieldType;
-    using ToFieldType = typename ToDataType::FieldType;
-    using ToNativeType = typename ToFieldType::NativeType;
 
-    static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template void convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);  \
+    extern template bool convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
 
-    if constexpr (std::is_floating_point_v<FromFieldType>)
-    {
-        if (!std::isfinite(value))
-        {
-            if constexpr (throw_exception)
-                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Cannot convert infinity or NaN to decimal", ToDataType::family_name);
-            else
-                return ReturnType(false);
-        }
-
-        auto out = value * static_cast<FromFieldType>(DecimalUtils::scaleMultiplier<ToNativeType>(scale));
-
-        if (out <= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::min()) ||
-            out >= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::max()))
-        {
-            if constexpr (throw_exception)
-                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Float is out of Decimal range", ToDataType::family_name);
-            else
-                return ReturnType(false);
-        }
-
-        result = static_cast<ToNativeType>(out);
-        return ReturnType(true);
-    }
-    else
-    {
-        if constexpr (is_big_int_v<FromFieldType>)
-            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal256>, ToDataType, ReturnType>(static_cast<Int256>(value), 0, scale, result));
-        else if constexpr (std::is_same_v<FromFieldType, UInt64>)
-            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal128>, ToDataType, ReturnType>(static_cast<Int128>(value), 0, scale, result));
-        else
-            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal64>, ToDataType, ReturnType>(static_cast<Int64>(value), 0, scale, result));
-    }
-}
 
 template <typename FromDataType, typename ToDataType>
 requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
-inline typename ToDataType::FieldType convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
+typename ToDataType::FieldType convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale);
-{
+
-    typename ToDataType::FieldType result;
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
-    convertToDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
+    extern template typename TO_DATA_TYPE::FieldType convertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
-    return result;
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
-}
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
 
 template <typename FromDataType, typename ToDataType>
 requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
-inline bool tryConvertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
+bool tryConvertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result);
-{
+
-    return convertToDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
-}
+    extern template bool tryConvertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
 
 template <typename T>
-inline DataTypePtr createDecimalMaxPrecision(UInt64 scale)
+DataTypePtr createDecimalMaxPrecision(UInt64 scale);
-{
+
-    return std::make_shared<DataTypeDecimal<T>>(DecimalUtils::max_precision<T>, scale);
+extern template DataTypePtr createDecimalMaxPrecision<Decimal32>(UInt64 scale);
-}
+extern template DataTypePtr createDecimalMaxPrecision<Decimal64>(UInt64 scale);
+extern template DataTypePtr createDecimalMaxPrecision<Decimal128>(UInt64 scale);
+extern template DataTypePtr createDecimalMaxPrecision<Decimal256>(UInt64 scale);
+
+extern template class DataTypeDecimal<Decimal32>;
+extern template class DataTypeDecimal<Decimal64>;
+extern template class DataTypeDecimal<Decimal128>;
+extern template class DataTypeDecimal<Decimal256>;
 
 }

src/DataTypes/DataTypesNumber.cpp

@@ -102,4 +102,21 @@ void registerDataTypeNumbers(DataTypeFactory & factory)
     factory.registerAlias("DOUBLE PRECISION", "Float64", DataTypeFactory::CaseInsensitive);
 }
 
+/// Explicit template instantiations.
+template class DataTypeNumber<UInt8>;
+template class DataTypeNumber<UInt16>;
+template class DataTypeNumber<UInt32>;
+template class DataTypeNumber<UInt64>;
+template class DataTypeNumber<Int8>;
+template class DataTypeNumber<Int16>;
+template class DataTypeNumber<Int32>;
+template class DataTypeNumber<Int64>;
+template class DataTypeNumber<Float32>;
+template class DataTypeNumber<Float64>;
+
+template class DataTypeNumber<UInt128>;
+template class DataTypeNumber<Int128>;
+template class DataTypeNumber<UInt256>;
+template class DataTypeNumber<Int256>;
+
 }

src/DataTypes/DataTypesNumber.h

@@ -55,6 +55,22 @@ private:
     bool unsigned_can_be_signed = false;
 };
 
+extern template class DataTypeNumber<UInt8>;
+extern template class DataTypeNumber<UInt16>;
+extern template class DataTypeNumber<UInt32>;
+extern template class DataTypeNumber<UInt64>;
+extern template class DataTypeNumber<Int8>;
+extern template class DataTypeNumber<Int16>;
+extern template class DataTypeNumber<Int32>;
+extern template class DataTypeNumber<Int64>;
+extern template class DataTypeNumber<Float32>;
+extern template class DataTypeNumber<Float64>;
+
+extern template class DataTypeNumber<UInt128>;
+extern template class DataTypeNumber<Int128>;
+extern template class DataTypeNumber<UInt256>;
+extern template class DataTypeNumber<Int256>;
+
 using DataTypeUInt8 = DataTypeNumber<UInt8>;
 using DataTypeUInt16 = DataTypeNumber<UInt16>;
 using DataTypeUInt32 = DataTypeNumber<UInt32>;

src/DataTypes/IDataType.cpp

@@ -267,4 +267,91 @@ SerializationPtr IDataType::getSerialization(const NameAndTypePair & column)
     return column.type->getDefaultSerialization();
 }
 
+#define FOR_TYPES_OF_TYPE(M) \
+    M(TypeIndex) \
+    M(const IDataType &) \
+    M(const DataTypePtr &) \
+    M(WhichDataType)
+
+#define DISPATCH(TYPE) \
+bool isUInt8(TYPE data_type) { return WhichDataType(data_type).isUInt8(); } \
+bool isUInt16(TYPE data_type) { return WhichDataType(data_type).isUInt16(); } \
+bool isUInt32(TYPE data_type) { return WhichDataType(data_type).isUInt32(); } \
+bool isUInt64(TYPE data_type) { return WhichDataType(data_type).isUInt64(); } \
+bool isNativeUInt(TYPE data_type) { return WhichDataType(data_type).isNativeUInt(); } \
+bool isUInt(TYPE data_type) { return WhichDataType(data_type).isUInt(); } \
+\
+bool isInt8(TYPE data_type) { return WhichDataType(data_type).isInt8(); } \
+bool isInt16(TYPE data_type) { return WhichDataType(data_type).isInt16(); } \
+bool isInt32(TYPE data_type) { return WhichDataType(data_type).isInt32(); } \
+bool isInt64(TYPE data_type) { return WhichDataType(data_type).isInt64(); } \
+bool isNativeInt(TYPE data_type) { return WhichDataType(data_type).isNativeInt(); } \
+bool isInt(TYPE data_type) { return WhichDataType(data_type).isInt(); } \
+\
+bool isInteger(TYPE data_type) { return WhichDataType(data_type).isInteger(); } \
+bool isNativeInteger(TYPE data_type) { return WhichDataType(data_type).isNativeInteger(); } \
+\
+bool isDecimal(TYPE data_type) { return WhichDataType(data_type).isDecimal(); } \
+\
+bool isFloat(TYPE data_type) { return WhichDataType(data_type).isFloat(); } \
+\
+bool isNativeNumber(TYPE data_type) { return WhichDataType(data_type).isNativeNumber(); } \
+bool isNumber(TYPE data_type) { return WhichDataType(data_type).isNumber(); } \
+\
+bool isEnum8(TYPE data_type) { return WhichDataType(data_type).isEnum8(); } \
+bool isEnum16(TYPE data_type) { return WhichDataType(data_type).isEnum16(); } \
+bool isEnum(TYPE data_type) { return WhichDataType(data_type).isEnum(); } \
+\
+bool isDate(TYPE data_type) { return WhichDataType(data_type).isDate(); } \
+bool isDate32(TYPE data_type) { return WhichDataType(data_type).isDate32(); } \
+bool isDateOrDate32(TYPE data_type) { return WhichDataType(data_type).isDateOrDate32(); } \
+bool isDateTime(TYPE data_type) { return WhichDataType(data_type).isDateTime(); } \
+bool isDateTime64(TYPE data_type) { return WhichDataType(data_type).isDateTime64(); } \
+bool isDateTimeOrDateTime64(TYPE data_type) { return WhichDataType(data_type).isDateTimeOrDateTime64(); } \
+bool isDateOrDate32OrDateTimeOrDateTime64(TYPE data_type) { return WhichDataType(data_type).isDateOrDate32OrDateTimeOrDateTime64(); } \
+\
+bool isString(TYPE data_type) { return WhichDataType(data_type).isString(); } \
+bool isFixedString(TYPE data_type) { return WhichDataType(data_type).isFixedString(); } \
+bool isStringOrFixedString(TYPE data_type) { return WhichDataType(data_type).isStringOrFixedString(); } \
+\
+bool isUUID(TYPE data_type) { return WhichDataType(data_type).isUUID(); } \
+bool isIPv4(TYPE data_type) { return WhichDataType(data_type).isIPv4(); } \
+bool isIPv6(TYPE data_type) { return WhichDataType(data_type).isIPv6(); } \
+bool isArray(TYPE data_type) { return WhichDataType(data_type).isArray(); } \
+bool isTuple(TYPE data_type) { return WhichDataType(data_type).isTuple(); } \
+bool isMap(TYPE data_type) {return WhichDataType(data_type).isMap(); } \
+bool isInterval(TYPE data_type) {return WhichDataType(data_type).isInterval(); } \
+bool isObject(TYPE data_type) { return WhichDataType(data_type).isObject(); } \
+bool isVariant(TYPE data_type) { return WhichDataType(data_type).isVariant(); } \
+bool isNothing(TYPE data_type) { return WhichDataType(data_type).isNothing(); } \
+\
+bool isColumnedAsNumber(TYPE data_type) \
+{ \
+    WhichDataType which(data_type); \
+    return which.isInteger() || which.isFloat() || which.isDateOrDate32OrDateTimeOrDateTime64() || which.isUUID() || which.isIPv4() || which.isIPv6(); \
+} \
+\
+bool isColumnedAsDecimal(TYPE data_type) \
+{ \
+    WhichDataType which(data_type); \
+    return which.isDecimal() || which.isDateTime64(); \
+} \
+\
+bool isNotCreatable(TYPE data_type) \
+{ \
+    WhichDataType which(data_type); \
+    return which.isNothing() || which.isFunction() || which.isSet(); \
+} \
+\
+bool isNotDecimalButComparableToDecimal(TYPE data_type) \
+{ \
+    WhichDataType which(data_type); \
+    return which.isInt() || which.isUInt() || which.isFloat(); \
+} \
+
+FOR_TYPES_OF_TYPE(DISPATCH)
+
+#undef DISPATCH
+#undef FOR_TYPES_OF_TYPE
+
 }

src/DataTypes/IDataType.h

@@ -424,71 +424,76 @@ struct WhichDataType
 
 /// IDataType helpers (alternative for IDataType virtual methods with single point of truth)
 
-template <typename T> inline bool isUInt8(const T & data_type) { return WhichDataType(data_type).isUInt8(); }
+#define FOR_TYPES_OF_TYPE(M) \
-template <typename T> inline bool isUInt16(const T & data_type) { return WhichDataType(data_type).isUInt16(); }
+    M(TypeIndex) \
-template <typename T> inline bool isUInt32(const T & data_type) { return WhichDataType(data_type).isUInt32(); }
+    M(const IDataType &) \
-template <typename T> inline bool isUInt64(const T & data_type) { return WhichDataType(data_type).isUInt64(); }
+    M(const DataTypePtr &) \
-template <typename T> inline bool isNativeUInt(const T & data_type) { return WhichDataType(data_type).isNativeUInt(); }
+    M(WhichDataType)
-template <typename T> inline bool isUInt(const T & data_type) { return WhichDataType(data_type).isUInt(); }
 
-template <typename T> inline bool isInt8(const T & data_type) { return WhichDataType(data_type).isInt8(); }
+#define DISPATCH(TYPE) \
-template <typename T> inline bool isInt16(const T & data_type) { return WhichDataType(data_type).isInt16(); }
+bool isUInt8(TYPE data_type); \
-template <typename T> inline bool isInt32(const T & data_type) { return WhichDataType(data_type).isInt32(); }
+bool isUInt16(TYPE data_type); \
-template <typename T> inline bool isInt64(const T & data_type) { return WhichDataType(data_type).isInt64(); }
+bool isUInt32(TYPE data_type); \
-template <typename T> inline bool isNativeInt(const T & data_type) { return WhichDataType(data_type).isNativeInt(); }
+bool isUInt64(TYPE data_type); \
-template <typename T> inline bool isInt(const T & data_type) { return WhichDataType(data_type).isInt(); }
+bool isNativeUInt(TYPE data_type); \
+bool isUInt(TYPE data_type); \
+\
+bool isInt8(TYPE data_type); \
+bool isInt16(TYPE data_type); \
+bool isInt32(TYPE data_type); \
+bool isInt64(TYPE data_type); \
+bool isNativeInt(TYPE data_type); \
+bool isInt(TYPE data_type); \
+\
+bool isInteger(TYPE data_type); \
+bool isNativeInteger(TYPE data_type); \
+\
+bool isDecimal(TYPE data_type); \
+\
+bool isFloat(TYPE data_type); \
+\
+bool isNativeNumber(TYPE data_type); \
+bool isNumber(TYPE data_type); \
+\
+bool isEnum8(TYPE data_type); \
+bool isEnum16(TYPE data_type); \
+bool isEnum(TYPE data_type); \
+\
+bool isDate(TYPE data_type); \
+bool isDate32(TYPE data_type); \
+bool isDateOrDate32(TYPE data_type); \
+bool isDateTime(TYPE data_type); \
+bool isDateTime64(TYPE data_type); \
+bool isDateTimeOrDateTime64(TYPE data_type); \
+bool isDateOrDate32OrDateTimeOrDateTime64(TYPE data_type); \
+\
+bool isString(TYPE data_type); \
+bool isFixedString(TYPE data_type); \
+bool isStringOrFixedString(TYPE data_type); \
+\
+bool isUUID(TYPE data_type); \
+bool isIPv4(TYPE data_type); \
+bool isIPv6(TYPE data_type); \
+bool isArray(TYPE data_type); \
+bool isTuple(TYPE data_type); \
+bool isMap(TYPE data_type); \
+bool isInterval(TYPE data_type); \
+bool isObject(TYPE data_type); \
+bool isVariant(TYPE data_type); \
+bool isNothing(TYPE data_type); \
+\
+bool isColumnedAsNumber(TYPE data_type); \
+\
+bool isColumnedAsDecimal(TYPE data_type); \
+\
+bool isNotCreatable(TYPE data_type); \
+\
+bool isNotDecimalButComparableToDecimal(TYPE data_type); \
 
-template <typename T> inline bool isInteger(const T & data_type) { return WhichDataType(data_type).isInteger(); }
+FOR_TYPES_OF_TYPE(DISPATCH)
-template <typename T> inline bool isNativeInteger(const T & data_type) { return WhichDataType(data_type).isNativeInteger(); }
 
-template <typename T> inline bool isDecimal(const T & data_type) { return WhichDataType(data_type).isDecimal(); }
+#undef DISPATCH
-
+#undef FOR_TYPES_OF_TYPE
-template <typename T> inline bool isFloat(const T & data_type) { return WhichDataType(data_type).isFloat(); }
-
-template <typename T> inline bool isNativeNumber(const T & data_type) { return WhichDataType(data_type).isNativeNumber(); }
-template <typename T> inline bool isNumber(const T & data_type) { return WhichDataType(data_type).isNumber(); }
-
-template <typename T> inline bool isEnum8(const T & data_type) { return WhichDataType(data_type).isEnum8(); }
-template <typename T> inline bool isEnum16(const T & data_type) { return WhichDataType(data_type).isEnum16(); }
-template <typename T> inline bool isEnum(const T & data_type) { return WhichDataType(data_type).isEnum(); }
-
-template <typename T> inline bool isDate(const T & data_type) { return WhichDataType(data_type).isDate(); }
-template <typename T> inline bool isDate32(const T & data_type) { return WhichDataType(data_type).isDate32(); }
-template <typename T> inline bool isDateOrDate32(const T & data_type) { return WhichDataType(data_type).isDateOrDate32(); }
-template <typename T> inline bool isDateTime(const T & data_type) { return WhichDataType(data_type).isDateTime(); }
-template <typename T> inline bool isDateTime64(const T & data_type) { return WhichDataType(data_type).isDateTime64(); }
-template <typename T> inline bool isDateTimeOrDateTime64(const T & data_type) { return WhichDataType(data_type).isDateTimeOrDateTime64(); }
-template <typename T> inline bool isDateOrDate32OrDateTimeOrDateTime64(const T & data_type) { return WhichDataType(data_type).isDateOrDate32OrDateTimeOrDateTime64(); }
-
-template <typename T> inline bool isString(const T & data_type) { return WhichDataType(data_type).isString(); }
-template <typename T> inline bool isFixedString(const T & data_type) { return WhichDataType(data_type).isFixedString(); }
-template <typename T> inline bool isStringOrFixedString(const T & data_type) { return WhichDataType(data_type).isStringOrFixedString(); }
-
-template <typename T> inline bool isUUID(const T & data_type) { return WhichDataType(data_type).isUUID(); }
-template <typename T> inline bool isIPv4(const T & data_type) { return WhichDataType(data_type).isIPv4(); }
-template <typename T> inline bool isIPv6(const T & data_type) { return WhichDataType(data_type).isIPv6(); }
-template <typename T> inline bool isArray(const T & data_type) { return WhichDataType(data_type).isArray(); }
-template <typename T> inline bool isTuple(const T & data_type) { return WhichDataType(data_type).isTuple(); }
-template <typename T> inline bool isMap(const T & data_type) {return WhichDataType(data_type).isMap(); }
-template <typename T> inline bool isInterval(const T & data_type) {return WhichDataType(data_type).isInterval(); }
-template <typename T> inline bool isObject(const T & data_type) { return WhichDataType(data_type).isObject(); }
-template <typename T> inline bool isVariant(const T & data_type) { return WhichDataType(data_type).isVariant(); }
-
-template <typename T> inline bool isNothing(const T & data_type) { return WhichDataType(data_type).isNothing(); }
-
-template <typename T>
-inline bool isColumnedAsNumber(const T & data_type)
-{
-    WhichDataType which(data_type);
-    return which.isInteger() || which.isFloat() || which.isDateOrDate32OrDateTimeOrDateTime64() || which.isUUID() || which.isIPv4() || which.isIPv6();
-}
-
-template <typename T>
-inline bool isColumnedAsDecimal(const T & data_type)
-{
-    WhichDataType which(data_type);
-    return which.isDecimal() || which.isDateTime64();
-}
 
 // Same as isColumnedAsDecimal but also checks value type of underlyig column.
 template <typename T, typename DataType>
@@ -498,19 +503,6 @@ inline bool isColumnedAsDecimalT(const DataType & data_type)
     return (which.isDecimal() || which.isDateTime64()) && which.idx == TypeToTypeIndex<T>;
 }
 
-template <typename T>
-inline bool isNotCreatable(const T & data_type)
-{
-    WhichDataType which(data_type);
-    return which.isNothing() || which.isFunction() || which.isSet();
-}
-
-inline bool isNotDecimalButComparableToDecimal(const DataTypePtr & data_type)
-{
-    WhichDataType which(data_type);
-    return which.isInt() || which.isUInt() || which.isFloat();
-}
-
 inline bool isBool(const DataTypePtr & data_type)
 {
     return data_type->getName() == "Bool";

src/DataTypes/Serializations/SerializationDecimalBase.h

@@ -29,4 +29,10 @@ public:
     void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override;
 };
 
+extern template class SerializationDecimalBase<Decimal32>;
+extern template class SerializationDecimalBase<Decimal64>;
+extern template class SerializationDecimalBase<Decimal128>;
+extern template class SerializationDecimalBase<Decimal256>;
+extern template class SerializationDecimalBase<DateTime64>;
+
 }

src/Dictionaries/HashedArrayDictionary.cpp

@@ -1078,7 +1078,7 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::calculateBytesAllocate
                     bytes_allocated += container.allocated_bytes();
                 }
 
-                bucket_count = container.capacity();
+                bucket_count += container.capacity();
             }
         };
 
@@ -1089,6 +1089,13 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::calculateBytesAllocate
                 bytes_allocated += container.size();
     }
 
+    /// `bucket_count` should be a sum over all shards,
+    /// but it should not be a sum over all attributes, since it is used to
+    /// calculate load_factor like this: `element_count / bucket_count`
+    /// While element_count is a sum over all shards, not over all attributes.
+    if (attributes.size())
+        bucket_count /= attributes.size();
+
     if (update_field_loaded_block)
         bytes_allocated += update_field_loaded_block->allocatedBytes();
 
@@ -1167,17 +1174,24 @@ void registerDictionaryArrayHashed(DictionaryFactory & factory)
         if (shards <= 0 || 128 < shards)
             throw Exception(ErrorCodes::BAD_ARGUMENTS,"{}: SHARDS parameter should be within [1, 128]", full_name);
 
-        HashedArrayDictionaryStorageConfiguration configuration{require_nonempty, dict_lifetime, static_cast<size_t>(shards)};
+        Int64 shard_load_queue_backlog = config.getInt(config_prefix + dictionary_layout_prefix + ".shard_load_queue_backlog", 10000);
+        if (shard_load_queue_backlog <= 0)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: SHARD_LOAD_QUEUE_BACKLOG parameter should be greater then zero", full_name);
 
         if (source_ptr->hasUpdateField() && shards > 1)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: SHARDS parameter does not supports for updatable source (UPDATE_FIELD)", full_name);
 
+        HashedArrayDictionaryStorageConfiguration configuration{require_nonempty, dict_lifetime, static_cast<size_t>(shards), static_cast<UInt64>(shard_load_queue_backlog)};
+
         ContextMutablePtr context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix);
         const auto & settings = context->getSettingsRef();
 
         const auto * clickhouse_source = dynamic_cast<const ClickHouseDictionarySource *>(source_ptr.get());
         configuration.use_async_executor = clickhouse_source && clickhouse_source->isLocal() && settings.dictionary_use_async_executor;
 
+        if (settings.max_execution_time.totalSeconds() > 0)
+            configuration.load_timeout = std::chrono::seconds(settings.max_execution_time.totalSeconds());
+
         if (dictionary_key_type == DictionaryKeyType::Simple)
         {
             if (shards > 1)

src/Dictionaries/HashedArrayDictionary.h

@@ -29,6 +29,7 @@ struct HashedArrayDictionaryStorageConfiguration
     size_t shards = 1;
     size_t shard_load_queue_backlog = 10000;
     bool use_async_executor = false;
+    std::chrono::seconds load_timeout{0};
 };
 
 template <DictionaryKeyType dictionary_key_type, bool sharded>

src/Dictionaries/HashedDictionary.h

@@ -67,6 +67,7 @@ struct HashedDictionaryConfiguration
     const bool require_nonempty;
     const DictionaryLifetime lifetime;
     bool use_async_executor = false;
+    const std::chrono::seconds load_timeout{0};
 };
 
 template <DictionaryKeyType dictionary_key_type, bool sparse, bool sharded>

src/Dictionaries/HashedDictionaryParallelLoader.h

@@ -31,6 +31,7 @@ template <DictionaryKeyType dictionary_key_type, bool sparse, bool sharded> clas
 namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
+    extern const int TIMEOUT_EXCEEDED;
 }
 
 }
@@ -50,9 +51,10 @@ public:
         , shards(dictionary.configuration.shards)
         , pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, CurrentMetrics::HashedDictionaryThreadsScheduled, shards)
         , shards_queues(shards)
+        , loading_timeout(dictionary.configuration.load_timeout)
     {
         UInt64 backlog = dictionary.configuration.shard_load_queue_backlog;
-        LOG_TRACE(dictionary.log, "Will load the {} dictionary using {} threads (with {} backlog)", dictionary_name, shards, backlog);
+        LOG_TRACE(dictionary.log, "Will load the {} dictionary using {} threads (with {} backlog and timeout {} sec)", dictionary_name, shards, backlog, loading_timeout.count());
 
         shards_slots.resize(shards);
         iota(shards_slots.data(), shards_slots.size(), UInt64(0));
@@ -62,7 +64,11 @@ public:
             shards_queues[shard].emplace(backlog);
             pool.scheduleOrThrowOnError([this, shard, thread_group = CurrentThread::getGroup()]
             {
+                WorkerStatistic statistic;
                 SCOPE_EXIT_SAFE(
+                    LOG_TRACE(dictionary.log, "Finished worker for dictionary {} shard {}, processed {} blocks, {} rows, total time {}ms",
+                        dictionary_name, shard, statistic.total_blocks, statistic.total_rows, statistic.total_elapsed_ms);
+
                     if (thread_group)
                         CurrentThread::detachFromGroupIfNotDetached();
                 );
@@ -74,7 +80,9 @@ public:
                     CurrentThread::attachToGroupIfDetached(thread_group);
                 setThreadName("HashedDictLoad");
 
-                threadWorker(shard);
+                LOG_TRACE(dictionary.log, "Starting worker for dictionary {}, shard {}", dictionary_name, shard);
+
+                threadWorker(shard, statistic);
             });
         }
     }
@@ -87,8 +95,28 @@ public:
 
         for (size_t shard = 0; shard < shards; ++shard)
         {
-            if (!shards_queues[shard]->push(std::move(shards_blocks[shard])))
+            const auto & current_block = shards_blocks[shard];
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not push to shards queue #{}", shard);
+            while (!shards_queues[shard]->tryPush(current_block, /* milliseconds= */ 100))
+            {
+                if (shards_queues[shard]->isFinished())
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not push to finished shards queue #{}, dictionary {}", shard, dictionary_name);
+
+                /// We need to check if some workers failed
+                if (pool.active() != shards)
+                {
+                    LOG_DEBUG(dictionary.log, "Some workers for dictionary {} failed, stopping all workers", dictionary_name);
+                    stop_all_workers = true;
+                    pool.wait(); /// We expect exception to be thrown from the failed worker thread
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Worker threads for dictionary {} are not active", dictionary_name);
+                }
+
+                if (loading_timeout.count() && std::chrono::milliseconds(total_loading_time.elapsedMilliseconds()) > loading_timeout)
+                {
+                    stop_all_workers = true;
+                    pool.wait();
+                    throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout {} sec for dictionary {} loading is expired", loading_timeout.count(), dictionary_name);
+                }
+            }
         }
     }
 
@@ -124,27 +152,49 @@ private:
     String dictionary_name;
     const size_t shards;
     ThreadPool pool;
+    std::atomic_bool stop_all_workers{false};
     std::vector<std::optional<ConcurrentBoundedQueue<Block>>> shards_queues;
+    std::chrono::seconds loading_timeout;
+    Stopwatch total_loading_time;
+
     std::vector<UInt64> shards_slots;
     DictionaryKeysArenaHolder<dictionary_key_type> arena_holder;
 
-    void threadWorker(size_t shard)
+    struct WorkerStatistic
+    {
+        UInt64 total_elapsed_ms = 0;
+        UInt64 total_blocks = 0;
+        UInt64 total_rows = 0;
+    };
+
+    void threadWorker(size_t shard, WorkerStatistic & statistic)
     {
         Block block;
         DictionaryKeysArenaHolder<dictionary_key_type> arena_holder_;
         auto & shard_queue = *shards_queues[shard];
 
-        while (shard_queue.pop(block))
+        while (true)
         {
+            if (!shard_queue.tryPop(block, /* milliseconds= */ 100))
+            {
+                /// Check if we need to stop
+                if (stop_all_workers || shard_queue.isFinished())
+                    break;
+                /// Timeout expired, but the queue is not finished yet, try again
+                continue;
+            }
+
             Stopwatch watch;
             dictionary.blockToAttributes(block, arena_holder_, shard);
             UInt64 elapsed_ms = watch.elapsedMilliseconds();
-            if (elapsed_ms > 1'000)
-                LOG_TRACE(dictionary.log, "Block processing for shard #{} is slow {}ms (rows {}).", shard, elapsed_ms, block.rows());
-        }
 
-        if (!shard_queue.isFinished())
+            statistic.total_elapsed_ms += elapsed_ms;
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not pull non finished shards queue #{}", shard);
+            statistic.total_blocks += 1;
+            statistic.total_rows += block.rows();
+
+            if (elapsed_ms > 1'000)
+                LOG_TRACE(dictionary.log, "Block processing for shard #{} is slow {}ms (rows {})", shard, elapsed_ms, block.rows());
+        }
     }
 
     /// Split block to shards smaller block, using 'selector'.

src/Dictionaries/registerHashedDictionary.cpp

@@ -77,6 +77,7 @@ void registerDictionaryHashed(DictionaryFactory & factory)
             require_nonempty,
             dict_lifetime,
             use_async_executor,
+            std::chrono::seconds(settings.max_execution_time.totalSeconds()),
         };
 
         if (source_ptr->hasUpdateField() && shards > 1)

src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp

@@ -23,7 +23,6 @@ namespace ErrorCodes
     extern const int CANNOT_OPEN_FILE;
     extern const int FILE_DOESNT_EXIST;
     extern const int BAD_FILE_TYPE;
-    extern const int FILE_ALREADY_EXISTS;
     extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED;
     extern const int LOGICAL_ERROR;
 }
@@ -593,14 +592,8 @@ void DiskObjectStorageTransaction::moveDirectory(const std::string & from_path,
 void DiskObjectStorageTransaction::moveFile(const String & from_path, const String & to_path)
 {
      operations_to_execute.emplace_back(
-        std::make_unique<PureMetadataObjectStorageOperation>(object_storage, metadata_storage, [from_path, to_path, this](MetadataTransactionPtr tx)
+        std::make_unique<PureMetadataObjectStorageOperation>(object_storage, metadata_storage, [from_path, to_path](MetadataTransactionPtr tx)
         {
-            if (metadata_storage.exists(to_path))
-                throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "File already exists: {}", to_path);
-
-            if (!metadata_storage.exists(from_path))
-                throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "File {} doesn't exist, cannot move", from_path);
-
             tx->moveFile(from_path, to_path);
         }));
 }

src/Functions/DateTimeTransforms.cpp

@@ -10,16 +10,17 @@ namespace ErrorCodes
 
 void throwDateIsNotSupported(const char * name)
 {
-    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type Date of argument for function {}", name);
+    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument of type Date for function {}", name);
-}
-
-void throwDateTimeIsNotSupported(const char * name)
-{
-    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type DateTime of argument for function {}", name);
 }
 
 void throwDate32IsNotSupported(const char * name)
 {
-    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type Date32 of argument for function {}", name);
+    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument of type Date32 for function {}", name);
 }
+
+void throwDateTimeIsNotSupported(const char * name)
+{
+    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument of type DateTime for function {}", name);
+}
+
 }

src/Functions/DateTimeTransforms.h

@@ -6,6 +6,7 @@
 #include <Common/DateLUTImpl.h>
 #include <Common/DateLUT.h>
 #include <Common/IntervalKind.h>
+#include "base/Decimal.h"
 #include <Columns/ColumnNullable.h>
 #include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnVector.h>
@@ -54,8 +55,8 @@ constexpr time_t MAX_DATE_TIMESTAMP = 5662310399;       // 2149-06-06 23:59:59 U
 constexpr time_t MAX_DATETIME_DAY_NUM =  49710;               // 2106-02-07
 
 [[noreturn]] void throwDateIsNotSupported(const char * name);
-[[noreturn]] void throwDateTimeIsNotSupported(const char * name);
 [[noreturn]] void throwDate32IsNotSupported(const char * name);
+[[noreturn]] void throwDateTimeIsNotSupported(const char * name);
 
 /// This factor transformation will say that the function is monotone everywhere.
 struct ZeroTransform
@@ -481,7 +482,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Nanosecond>
     }
     static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
     {
-        throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
+        throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
     }
     static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64)
     {
@@ -516,7 +517,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Microsecond>
     }
     static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
     {
-        throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
+        throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
     }
     static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64)
     {
@@ -559,7 +560,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Millisecond>
     }
     static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
     {
-        throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
+        throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
     }
     static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64)
     {
@@ -602,7 +603,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Second>
     }
     static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
     {
-        throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
+        throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
     }
     static UInt32 execute(UInt32 t, Int64 seconds, const DateLUTImpl & time_zone, Int64)
     {
@@ -623,7 +624,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Minute>
     }
     static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
     {
-        throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
+        throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
     }
     static UInt32 execute(UInt32 t, Int64 minutes, const DateLUTImpl & time_zone, Int64)
     {
@@ -644,7 +645,7 @@ struct ToStartOfInterval<IntervalKind::Kind::Hour>
     }
     static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64)
     {
-        throwDateIsNotSupported(TO_START_OF_INTERVAL_NAME);
+        throwDate32IsNotSupported(TO_START_OF_INTERVAL_NAME);
     }
     static UInt32 execute(UInt32 t, Int64 hours, const DateLUTImpl & time_zone, Int64)
     {
@@ -777,7 +778,7 @@ struct ToTimeImpl
     }
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt32 execute(UInt16, const DateLUTImpl &)
     {
@@ -802,7 +803,7 @@ struct ToStartOfMinuteImpl
     }
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt32 execute(UInt16, const DateLUTImpl &)
     {
@@ -849,7 +850,7 @@ struct ToStartOfSecondImpl
     }
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt32 execute(UInt16, const DateLUTImpl &)
     {
@@ -897,7 +898,7 @@ struct ToStartOfMillisecondImpl
     }
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt32 execute(UInt16, const DateLUTImpl &)
     {
@@ -941,7 +942,7 @@ struct ToStartOfMicrosecondImpl
     }
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt32 execute(UInt16, const DateLUTImpl &)
     {
@@ -979,7 +980,7 @@ struct ToStartOfNanosecondImpl
     }
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt32 execute(UInt16, const DateLUTImpl &)
     {
@@ -1004,7 +1005,7 @@ struct ToStartOfFiveMinutesImpl
     }
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt32 execute(UInt16, const DateLUTImpl &)
     {
@@ -1036,7 +1037,7 @@ struct ToStartOfTenMinutesImpl
     }
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt32 execute(UInt16, const DateLUTImpl &)
     {
@@ -1068,7 +1069,7 @@ struct ToStartOfFifteenMinutesImpl
     }
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt32 execute(UInt16, const DateLUTImpl &)
     {
@@ -1103,7 +1104,7 @@ struct TimeSlotImpl
 
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
 
     static UInt32 execute(UInt16, const DateLUTImpl &)
@@ -1142,7 +1143,7 @@ struct ToStartOfHourImpl
 
     static UInt32 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
 
     static UInt32 execute(UInt16, const DateLUTImpl &)
@@ -1429,7 +1430,7 @@ struct ToHourImpl
     }
     static UInt8 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt8 execute(UInt16, const DateLUTImpl &)
     {
@@ -1456,7 +1457,7 @@ struct TimezoneOffsetImpl
 
     static time_t execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
 
     static time_t execute(UInt16, const DateLUTImpl &)
@@ -1482,7 +1483,7 @@ struct ToMinuteImpl
     }
     static UInt8 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt8 execute(UInt16, const DateLUTImpl &)
     {
@@ -1507,7 +1508,7 @@ struct ToSecondImpl
     }
     static UInt8 execute(Int32, const DateLUTImpl &)
     {
-        throwDateIsNotSupported(name);
+        throwDate32IsNotSupported(name);
     }
     static UInt8 execute(UInt16, const DateLUTImpl &)
     {
@@ -1518,6 +1519,32 @@ struct ToSecondImpl
     using FactorTransform = ToStartOfMinuteImpl;
 };
 
+struct ToMillisecondImpl
+{
+    static constexpr auto name = "toMillisecond";
+
+    static UInt16 execute(const DateTime64 & datetime64, Int64 scale_multiplier, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toMillisecond<DateTime64>(datetime64, scale_multiplier);
+    }
+
+    static UInt16 execute(UInt32, const DateLUTImpl &)
+    {
+        return 0;
+    }
+    static UInt16 execute(Int32, const DateLUTImpl &)
+    {
+        throwDate32IsNotSupported(name);
+    }
+    static UInt16 execute(UInt16, const DateLUTImpl &)
+    {
+        throwDateIsNotSupported(name);
+    }
+    static constexpr bool hasPreimage() { return false; }
+
+    using FactorTransform = ZeroTransform;
+};
+
 struct ToISOYearImpl
 {
     static constexpr auto name = "toISOYear";

src/Functions/FunctionBase64Conversion.h

@@ -100,7 +100,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors mandatory_arguments{
-            {"value", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"}
+            {"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_arguments);

src/Functions/FunctionHelpers.h

@@ -108,8 +108,10 @@ struct FunctionArgumentDescriptor
 {
     const char * argument_name;
 
-    std::function<bool (const IDataType &)> type_validator_func;
+    using TypeValidator = bool (*)(const IDataType &);
-    std::function<bool (const IColumn &)> column_validator_func;
+    TypeValidator type_validator_func;
+    using ColumnValidator = bool (*)(const IColumn &);
+    ColumnValidator column_validator_func;
 
     const char * expected_type_description;
 

src/Functions/FunctionStringReplace.h

@@ -35,9 +35,9 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"haystack", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"},
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"},
-            {"pattern", &isString<IDataType>, nullptr, "String"},
+            {"pattern", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
-            {"replacement", &isString<IDataType>, nullptr, "String"}
+            {"replacement", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, args);

src/Functions/FunctionTokens.h

@@ -74,6 +74,8 @@ public:
 
     size_t getNumberOfArguments() const override { return Generator::getNumberOfArguments(); }
 
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return Generator::getArgumentsThatAreAlwaysConstant(); }
+
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         Generator::checkArguments(*this, arguments);
@@ -184,12 +186,12 @@ static inline void checkArgumentsWithSeparatorAndOptionalMaxSubstrings(
     const IFunction & func, const ColumnsWithTypeAndName & arguments)
 {
     FunctionArgumentDescriptors mandatory_args{
-        {"separator", &isString<IDataType>, isColumnConst, "const String"},
+        {"separator", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"},
-        {"s", &isString<IDataType>, nullptr, "String"}
+        {"s", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
     };
 
     FunctionArgumentDescriptors optional_args{
-        {"max_substrings", &isNativeInteger<IDataType>, isColumnConst, "const Number"},
+        {"max_substrings", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), isColumnConst, "const Number"},
     };
 
     validateFunctionArgumentTypes(func, arguments, mandatory_args, optional_args);
@@ -198,11 +200,11 @@ static inline void checkArgumentsWithSeparatorAndOptionalMaxSubstrings(
 static inline void checkArgumentsWithOptionalMaxSubstrings(const IFunction & func, const ColumnsWithTypeAndName & arguments)
 {
     FunctionArgumentDescriptors mandatory_args{
-        {"s", &isString<IDataType>, nullptr, "String"},
+        {"s", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
     };
 
     FunctionArgumentDescriptors optional_args{
-        {"max_substrings", &isNativeInteger<IDataType>, isColumnConst, "const Number"},
+        {"max_substrings", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), isColumnConst, "const Number"},
     };
 
     validateFunctionArgumentTypes(func, arguments, mandatory_args, optional_args);

src/Functions/FunctionUnixTimestamp64.h

@@ -45,7 +45,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"value", &isDateTime64<IDataType>, nullptr, "DateTime64"}
+            {"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime64), nullptr, "DateTime64"}
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/FunctionsAES.h

@@ -154,21 +154,21 @@ private:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         auto optional_args = FunctionArgumentDescriptors{
-            {"IV", &isStringOrFixedString<IDataType>, nullptr, "Initialization vector binary string"},
+            {"IV", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Initialization vector binary string"},
         };
 
         if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL)
         {
             optional_args.emplace_back(FunctionArgumentDescriptor{
-                "AAD", &isStringOrFixedString<IDataType>, nullptr, "Additional authenticated data binary string for GCM mode"
+                "AAD", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Additional authenticated data binary string for GCM mode"
             });
         }
 
         validateFunctionArgumentTypes(*this, arguments,
             FunctionArgumentDescriptors{
-                {"mode", &isStringOrFixedString<IDataType>, isColumnConst, "encryption mode string"},
+                {"mode", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "encryption mode string"},
-                {"input", &isStringOrFixedString<IDataType>, {}, "plaintext"},
+                {"input", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "plaintext"},
-                {"key", &isStringOrFixedString<IDataType>, {}, "encryption key binary string"},
+                {"key", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "encryption key binary string"},
             },
             optional_args
         );
@@ -425,21 +425,21 @@ private:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         auto optional_args = FunctionArgumentDescriptors{
-            {"IV", &isStringOrFixedString<IDataType>, nullptr, "Initialization vector binary string"},
+            {"IV", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Initialization vector binary string"},
         };
 
         if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL)
         {
             optional_args.emplace_back(FunctionArgumentDescriptor{
-                "AAD", &isStringOrFixedString<IDataType>, nullptr, "Additional authenticated data binary string for GCM mode"
+                "AAD", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Additional authenticated data binary string for GCM mode"
             });
         }
 
         validateFunctionArgumentTypes(*this, arguments,
             FunctionArgumentDescriptors{
-                {"mode", &isStringOrFixedString<IDataType>, isColumnConst, "decryption mode string"},
+                {"mode", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "decryption mode string"},
-                {"input", &isStringOrFixedString<IDataType>, {}, "ciphertext"},
+                {"input", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "ciphertext"},
-                {"key", &isStringOrFixedString<IDataType>, {}, "decryption key binary string"},
+                {"key", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "decryption key binary string"},
             },
             optional_args
         );

src/Functions/FunctionsConversion.h

@@ -2129,12 +2129,12 @@ public:
 
         if constexpr (to_decimal)
         {
-            mandatory_args.push_back({"scale", &isNativeInteger<IDataType>, &isColumnConst, "const Integer"});
+            mandatory_args.push_back({"scale", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), &isColumnConst, "const Integer"});
         }
 
         if (!to_decimal && isDateTime64<Name, ToDataType>(arguments))
         {
-            mandatory_args.push_back({"scale", &isNativeInteger<IDataType>, &isColumnConst, "const Integer"});
+            mandatory_args.push_back({"scale", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), &isColumnConst, "const Integer"});
         }
 
         // toString(DateTime or DateTime64, [timezone: String])
@@ -2150,7 +2150,7 @@ public:
             // toDateTime64(value, scale : Integer[, timezone: String])
             || std::is_same_v<ToDataType, DataTypeDateTime64>)
         {
-            optional_args.push_back({"timezone", &isString<IDataType>, nullptr, "String"});
+            optional_args.push_back({"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"});
         }
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
@@ -2498,11 +2498,11 @@ public:
         if (isDateTime64<Name, ToDataType>(arguments))
         {
             validateFunctionArgumentTypes(*this, arguments,
-                FunctionArgumentDescriptors{{"string", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"}},
+                FunctionArgumentDescriptors{{"string", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"}},
                 // optional
                 FunctionArgumentDescriptors{
-                    {"precision", &isUInt8<IDataType>, isColumnConst, "const UInt8"},
+                    {"precision", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt8), isColumnConst, "const UInt8"},
-                    {"timezone", &isStringOrFixedString<IDataType>, isColumnConst, "const String or FixedString"},
+                    {"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "const String or FixedString"},
                 });
 
             UInt64 scale = to_datetime64 ? DataTypeDateTime64::default_scale : 0;

src/Functions/JSONArrayLength.cpp

@@ -45,7 +45,7 @@ namespace
         DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
         {
             auto args = FunctionArgumentDescriptors{
-                {"json", &isString<IDataType>, nullptr, "String"},
+                {"json", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
             };
 
             validateFunctionArgumentTypes(*this, arguments, args);

src/Functions/URL/URLHierarchy.cpp

@@ -24,10 +24,12 @@ public:
     static bool isVariadic() { return false; }
     static size_t getNumberOfArguments() { return 1; }
 
+    static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {}; }
+
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"URL", &isString<IDataType>, nullptr, "String"},
+            {"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
         };
 
         validateFunctionArgumentTypes(func, arguments, mandatory_args);

src/Functions/URL/URLPathHierarchy.cpp

@@ -22,10 +22,12 @@ public:
     static bool isVariadic() { return false; }
     static size_t getNumberOfArguments() { return 1; }
 
+    static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {}; }
+
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"URL", &isString<IDataType>, nullptr, "String"},
+            {"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
         };
 
         validateFunctionArgumentTypes(func, arguments, mandatory_args);

src/Functions/URL/extractURLParameterNames.cpp

@@ -22,10 +22,12 @@ public:
     static bool isVariadic() { return false; }
     static size_t getNumberOfArguments() { return 1; }
 
+    static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {}; }
+
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"URL", &isString<IDataType>, nullptr, "String"},
+            {"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
         };
 
         validateFunctionArgumentTypes(func, arguments, mandatory_args);

src/Functions/URL/extractURLParameters.cpp

@@ -23,10 +23,12 @@ public:
     static bool isVariadic() { return false; }
     static size_t getNumberOfArguments() { return 1; }
 
+    static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {}; }
+
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"URL", &isString<IDataType>, nullptr, "String"},
+            {"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
         };
 
         validateFunctionArgumentTypes(func, arguments, mandatory_args);

src/Functions/alphaTokens.cpp

@@ -32,6 +32,8 @@ public:
 
     static size_t getNumberOfArguments() { return 0; }
 
+    static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1}; }
+
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         checkArgumentsWithOptionalMaxSubstrings(func, arguments);

src/Functions/array/arrayJaccardIndex.cpp

@@ -84,8 +84,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"array_1", &isArray<IDataType>, nullptr, "Array"},
+            {"array_1", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
-            {"array_2", &isArray<IDataType>, nullptr, "Array"},
+            {"array_2", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
         };
         validateFunctionArgumentTypes(*this, arguments, args);
         return std::make_shared<DataTypeNumber<ResultType>>();

src/Functions/array/arrayRandomSample.cpp

@@ -36,8 +36,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"array", &isArray<IDataType>, nullptr, "Array"},
+            {"array", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
-            {"samples", &isUInt<IDataType>, isColumnConst, "const UInt*"},
+            {"samples", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt), isColumnConst, "const UInt*"},
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/array/arrayShingles.cpp

@@ -28,8 +28,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
             FunctionArgumentDescriptors args{
-                {"array", &isArray<IDataType>, nullptr, "Array"},
+                {"array", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
-                {"length", &isInteger<IDataType>, nullptr, "Integer"}
+                {"length", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isInteger), nullptr, "Integer"}
             };
             validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/arrayStringConcat.cpp

@@ -151,12 +151,12 @@ public:
     {
         FunctionArgumentDescriptors mandatory_args
         {
-            {"arr", &isArray<IDataType>, nullptr, "Array"},
+            {"arr", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
         };
 
         FunctionArgumentDescriptors optional_args
         {
-            {"separator", &isString<IDataType>, isColumnConst, "const String"},
+            {"separator", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"},
         };
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);

src/Functions/castOrDefault.cpp

@@ -210,10 +210,10 @@ private:
         FunctionArgumentDescriptors optional_args;
 
         if constexpr (IsDataTypeDecimal<Type>)
-            mandatory_args.push_back({"scale", &isNativeInteger<IDataType>, &isColumnConst, "const Integer"});
+            mandatory_args.push_back({"scale", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), &isColumnConst, "const Integer"});
 
         if (std::is_same_v<Type, DataTypeDateTime> || std::is_same_v<Type, DataTypeDateTime64>)
-            optional_args.push_back({"timezone", &isString<IDataType>, isColumnConst, "const String"});
+            optional_args.push_back({"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"});
 
         optional_args.push_back({"default_value", nullptr, nullptr, nullptr});
 

src/Functions/countMatches.h

@@ -35,8 +35,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"haystack", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"},
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"},
-            {"pattern", &isString<IDataType>, isColumnConst, "constant String"}
+            {"pattern", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "constant String"}
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/extractAll.cpp

@@ -50,11 +50,13 @@ public:
     static bool isVariadic() { return false; }
     static size_t getNumberOfArguments() { return 2; }
 
+    static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1}; }
+
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"haystack", &isString<IDataType>, nullptr, "String"},
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
-            {"pattern", &isString<IDataType>, isColumnConst, "const String"}
+            {"pattern", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"}
         };
 
         validateFunctionArgumentTypes(func, arguments, mandatory_args);

src/Functions/extractAllGroups.h

@@ -71,8 +71,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"haystack", &isStringOrFixedString<IDataType>, nullptr, "const String or const FixedString"},
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "const String or const FixedString"},
-            {"needle", &isStringOrFixedString<IDataType>, isColumnConst, "const String or const FixedString"},
+            {"needle", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "const String or const FixedString"},
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/extractGroups.cpp

@@ -45,8 +45,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"haystack", &isStringOrFixedString<IDataType>, nullptr, "const String or const FixedString"},
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "const String or const FixedString"},
-            {"needle", &isStringOrFixedString<IDataType>, isColumnConst, "const String or const FixedString"},
+            {"needle", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "const String or const FixedString"},
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/formatQuery.cpp

@@ -54,7 +54,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"query", &isString<IDataType>, nullptr, "String"}
+            {"query", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
         };
         validateFunctionArgumentTypes(*this, arguments, args);