Merge remote-tracking branch 'origin/master' into fix-usan-parallel-replicas-prewhere

.github/workflows/master.yml

@@ -305,7 +305,7 @@ jobs:
       runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
   MarkReleaseReady:
-    if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
+    if: ${{ !failure() && !cancelled() }}
     needs:
       - BuilderBinDarwin
       - BuilderBinDarwinAarch64
@@ -313,9 +313,25 @@ jobs:
       - BuilderDebAarch64
     runs-on: [self-hosted, style-checker]
     steps:
+      - name: Debug
+        run: |
+          echo need with different filters
+          cat << 'EOF'
+          ${{ toJSON(needs) }}
+          ${{ toJSON(needs.*.result) }}
+          no failures ${{ !contains(needs.*.result, 'failure') }}
+          no skips ${{ !contains(needs.*.result, 'skipped') }}
+          no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
+          EOF
+      - name: Not ready
+        # fail the job to be able restart it
+        if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
+        run: exit 1
       - name: Check out repository code
+        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
         uses: ClickHouse/checkout@v1
       - name: Mark Commit Release Ready
+        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 mark_release_ready.py

.github/workflows/release_branches.yml

@@ -206,7 +206,7 @@ jobs:
       runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
   MarkReleaseReady:
-    if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
+    if: ${{ !failure() && !cancelled() }}
     needs:
       - BuilderBinDarwin
       - BuilderBinDarwinAarch64
@@ -214,9 +214,25 @@ jobs:
       - BuilderDebAarch64
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
+      - name: Debug
+        run: |
+          echo need with different filters
+          cat << 'EOF'
+          ${{ toJSON(needs) }}
+          ${{ toJSON(needs.*.result) }}
+          no failures ${{ !contains(needs.*.result, 'failure') }}
+          no skips ${{ !contains(needs.*.result, 'skipped') }}
+          no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
+          EOF
+      - name: Not ready
+        # fail the job to be able restart it
+        if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
+        run: exit 1
       - name: Check out repository code
+        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
         uses: ClickHouse/checkout@v1
       - name: Mark Commit Release Ready
+        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 mark_release_ready.py

.github/workflows/reusable_build.yml

@@ -43,7 +43,8 @@ jobs:
     runs-on: [self-hosted, '${{inputs.runner_type}}']
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        # WIP: temporary try commit with limited perallelization of checkout
+        uses: ClickHouse/checkout@0be3f7b3098bae494d3ef5d29d2e0676fb606232
         with:
           clear-repository: true
           ref: ${{ fromJson(inputs.data).git_ref }}

README.md

@@ -31,15 +31,30 @@ curl https://clickhouse.com/ | sh
 * [Static Analysis (SonarCloud)](https://sonarcloud.io/project/issues?resolved=false&id=ClickHouse_ClickHouse) proposes C++ quality improvements.
 * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
 
+## Monthly Release & Community Call
+
+Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know.
+
+* [v24.3 Community Call](https://clickhouse.com/company/events/v24-3-community-release-call) - Mar 26
+* [v24.4 Community Call](https://clickhouse.com/company/events/v24-4-community-release-call) - Apr 30
+
 ## Upcoming Events
 
-Keep an eye out for upcoming meetups around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com.
+Keep an eye out for upcoming meetups and eventsaround the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
+
+* [ClickHouse Meetup in Bellevue](https://www.meetup.com/clickhouse-seattle-user-group/events/298650371/) - Mar 11
+* [ClickHouse Meetup at Ramp's Offices in NYC](https://www.meetup.com/clickhouse-new-york-user-group/events/298640542/) - Mar 19
+* [ClickHouse Melbourne Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/299479750/) - Mar 20
+* [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/298997115/) - Mar 21
+* [ClickHouse Meetup in Bengaluru](https://www.meetup.com/clickhouse-bangalore-user-group/events/299479850/) - Mar 23
+* [ClickHouse Meetup in Zurich](https://www.meetup.com/clickhouse-switzerland-meetup-group/events/299628922/) - Apr 16
+* [ClickHouse Meetup in Copenhagen](https://www.meetup.com/clickhouse-denmark-meetup-group/events/299629133/) - Apr 23
+* [ClickHouse Meetup in Dubai](https://www.meetup.com/clickhouse-dubai-meetup-group/events/299629189/) - May 28
+
 
 ## Recent Recordings
 * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
-* **Recording available**: [**v24.1 Release Webinar**](https://www.youtube.com/watch?v=pBF9g0wGAGs) All the features of 24.1, one convenient video! Watch it now!
-* **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)
-
+* **Recording available**: [**v24.2 Release Call**](https://www.youtube.com/watch?v=iN2y-TK8f3A) All the features of 24.2, one convenient video! Watch it now!
   
  ## Interested in joining ClickHouse and making it your full-time job? 
   

base/base/CMakeLists.txt

@@ -13,6 +13,7 @@ set (SRCS
     cgroupsv2.cpp
     coverage.cpp
     demangle.cpp
+    Decimal.cpp
     getAvailableMemoryAmount.cpp
     getFQDNOrHostName.cpp
     getMemoryAmount.cpp

base/base/Decimal.cpp

@@ -0,0 +1,87 @@
+#include <base/Decimal.h>
+#include <base/extended_types.h>
+
+namespace DB
+{
+
+/// Explicit template instantiations.
+
+#define FOR_EACH_UNDERLYING_DECIMAL_TYPE(M) \
+    M(Int32)  \
+    M(Int64)  \
+    M(Int128) \
+    M(Int256)
+
+#define FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(M, X) \
+    M(Int32, X) \
+    M(Int64, X) \
+    M(Int128, X) \
+    M(Int256, X)
+
+template <typename T> const Decimal<T> & Decimal<T>::operator += (const T & x) { value += x; return *this; }
+template <typename T> const Decimal<T> & Decimal<T>::operator -= (const T & x) { value -= x; return *this; }
+template <typename T> const Decimal<T> & Decimal<T>::operator *= (const T & x) { value *= x; return *this; }
+template <typename T> const Decimal<T> & Decimal<T>::operator /= (const T & x) { value /= x; return *this; }
+template <typename T> const Decimal<T> & Decimal<T>::operator %= (const T & x) { value %= x; return *this; }
+
+template <typename T> void NO_SANITIZE_UNDEFINED Decimal<T>::addOverflow(const T & x) { value += x; }
+
+/// Maybe this explicit instantiation affects performance since operators cannot be inlined.
+
+template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator += (const Decimal<U> & x) { value += static_cast<T>(x.value); return *this; }
+template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator -= (const Decimal<U> & x) { value -= static_cast<T>(x.value); return *this; }
+template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator *= (const Decimal<U> & x) { value *= static_cast<T>(x.value); return *this; }
+template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator /= (const Decimal<U> & x) { value /= static_cast<T>(x.value); return *this; }
+template <typename T> template <typename U> const Decimal<T> & Decimal<T>::operator %= (const Decimal<U> & x) { value %= static_cast<T>(x.value); return *this; }
+
+#define DISPATCH(TYPE_T, TYPE_U) \
+    template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator += (const Decimal<TYPE_U> & x); \
+    template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator -= (const Decimal<TYPE_U> & x); \
+    template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator *= (const Decimal<TYPE_U> & x); \
+    template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator /= (const Decimal<TYPE_U> & x); \
+    template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator %= (const Decimal<TYPE_U> & x);
+#define INVOKE(X) FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+#define DISPATCH(TYPE) template struct Decimal<TYPE>;
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
+
+template <typename T> bool operator< (const Decimal<T> & x, const Decimal<T> & y) { return x.value < y.value; }
+template <typename T> bool operator> (const Decimal<T> & x, const Decimal<T> & y) { return x.value > y.value; }
+template <typename T> bool operator<= (const Decimal<T> & x, const Decimal<T> & y) { return x.value <= y.value; }
+template <typename T> bool operator>= (const Decimal<T> & x, const Decimal<T> & y) { return x.value >= y.value; }
+template <typename T> bool operator== (const Decimal<T> & x, const Decimal<T> & y) { return x.value == y.value; }
+template <typename T> bool operator!= (const Decimal<T> & x, const Decimal<T> & y) { return x.value != y.value; }
+
+#define DISPATCH(TYPE) \
+template bool operator< (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template bool operator> (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template bool operator<= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template bool operator>= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template bool operator== (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template bool operator!= (const Decimal<TYPE> & x, const Decimal<TYPE> & y);
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
+
+
+template <typename T> Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y) { return x.value + y.value; }
+template <typename T> Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y) { return x.value - y.value; }
+template <typename T> Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y) { return x.value * y.value; }
+template <typename T> Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y) { return x.value / y.value; }
+template <typename T> Decimal<T> operator- (const Decimal<T> & x) { return -x.value; }
+
+#define DISPATCH(TYPE) \
+template Decimal<TYPE> operator+ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template Decimal<TYPE> operator- (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template Decimal<TYPE> operator* (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template Decimal<TYPE> operator/ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+template Decimal<TYPE> operator- (const Decimal<TYPE> & x);
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
+
+#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS
+#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE
+}

base/base/Decimal.h

@@ -2,6 +2,7 @@
 
 #include <base/extended_types.h>
 #include <base/Decimal_fwd.h>
+#include <base/types.h>
 #include <base/defines.h>
 
 
@@ -10,6 +11,18 @@ namespace DB
 template <class> struct Decimal;
 class DateTime64;
 
+#define FOR_EACH_UNDERLYING_DECIMAL_TYPE(M) \
+    M(Int32) \
+    M(Int64) \
+    M(Int128) \
+    M(Int256)
+
+#define FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(M, X) \
+    M(Int32,  X) \
+    M(Int64,  X) \
+    M(Int128, X) \
+    M(Int256, X)
+
 using Decimal32 = Decimal<Int32>;
 using Decimal64 = Decimal<Int64>;
 using Decimal128 = Decimal<Int128>;
@@ -50,36 +63,73 @@ struct Decimal
             return static_cast<U>(value);
     }
 
-    const Decimal<T> & operator += (const T & x) { value += x; return *this; }
-    const Decimal<T> & operator -= (const T & x) { value -= x; return *this; }
-    const Decimal<T> & operator *= (const T & x) { value *= x; return *this; }
-    const Decimal<T> & operator /= (const T & x) { value /= x; return *this; }
-    const Decimal<T> & operator %= (const T & x) { value %= x; return *this; }
+    const Decimal<T> & operator += (const T & x);
+    const Decimal<T> & operator -= (const T & x);
+    const Decimal<T> & operator *= (const T & x);
+    const Decimal<T> & operator /= (const T & x);
+    const Decimal<T> & operator %= (const T & x);
 
-    template <typename U> const Decimal<T> & operator += (const Decimal<U> & x) { value += x.value; return *this; }
-    template <typename U> const Decimal<T> & operator -= (const Decimal<U> & x) { value -= x.value; return *this; }
-    template <typename U> const Decimal<T> & operator *= (const Decimal<U> & x) { value *= x.value; return *this; }
-    template <typename U> const Decimal<T> & operator /= (const Decimal<U> & x) { value /= x.value; return *this; }
-    template <typename U> const Decimal<T> & operator %= (const Decimal<U> & x) { value %= x.value; return *this; }
+    template <typename U> const Decimal<T> & operator += (const Decimal<U> & x);
+    template <typename U> const Decimal<T> & operator -= (const Decimal<U> & x);
+    template <typename U> const Decimal<T> & operator *= (const Decimal<U> & x);
+    template <typename U> const Decimal<T> & operator /= (const Decimal<U> & x);
+    template <typename U> const Decimal<T> & operator %= (const Decimal<U> & x);
 
     /// This is to avoid UB for sumWithOverflow()
-    void NO_SANITIZE_UNDEFINED addOverflow(const T & x) { value += x; }
+    void NO_SANITIZE_UNDEFINED addOverflow(const T & x);
 
     T value;
 };
 
-template <typename T> inline bool operator< (const Decimal<T> & x, const Decimal<T> & y) { return x.value < y.value; }
-template <typename T> inline bool operator> (const Decimal<T> & x, const Decimal<T> & y) { return x.value > y.value; }
-template <typename T> inline bool operator<= (const Decimal<T> & x, const Decimal<T> & y) { return x.value <= y.value; }
-template <typename T> inline bool operator>= (const Decimal<T> & x, const Decimal<T> & y) { return x.value >= y.value; }
-template <typename T> inline bool operator== (const Decimal<T> & x, const Decimal<T> & y) { return x.value == y.value; }
-template <typename T> inline bool operator!= (const Decimal<T> & x, const Decimal<T> & y) { return x.value != y.value; }
+#define DISPATCH(TYPE) extern template struct Decimal<TYPE>;
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
 
-template <typename T> inline Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y) { return x.value + y.value; }
-template <typename T> inline Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y) { return x.value - y.value; }
-template <typename T> inline Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y) { return x.value * y.value; }
-template <typename T> inline Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y) { return x.value / y.value; }
-template <typename T> inline Decimal<T> operator- (const Decimal<T> & x) { return -x.value; }
+#define DISPATCH(TYPE_T, TYPE_U) \
+    extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator += (const Decimal<TYPE_U> & x); \
+    extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator -= (const Decimal<TYPE_U> & x); \
+    extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator *= (const Decimal<TYPE_U> & x); \
+    extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator /= (const Decimal<TYPE_U> & x); \
+    extern template const Decimal<TYPE_T> & Decimal<TYPE_T>::operator %= (const Decimal<TYPE_U> & x);
+#define INVOKE(X) FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+template <typename T> bool operator< (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> bool operator> (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> bool operator<= (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> bool operator>= (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> bool operator== (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> bool operator!= (const Decimal<T> & x, const Decimal<T> & y);
+
+#define DISPATCH(TYPE) \
+extern template bool operator< (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template bool operator> (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template bool operator<= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template bool operator>= (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template bool operator== (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template bool operator!= (const Decimal<TYPE> & x, const Decimal<TYPE> & y);
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
+
+template <typename T> Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y);
+template <typename T> Decimal<T> operator- (const Decimal<T> & x);
+
+#define DISPATCH(TYPE) \
+extern template Decimal<TYPE> operator+ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template Decimal<TYPE> operator- (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template Decimal<TYPE> operator* (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template Decimal<TYPE> operator/ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
+extern template Decimal<TYPE> operator- (const Decimal<TYPE> & x);
+FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
+#undef DISPATCH
+
+#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE_PASS
+#undef FOR_EACH_UNDERLYING_DECIMAL_TYPE
 
 /// Distinguishable type to allow function resolution/deduction based on value type,
 /// but also relatively easy to convert to/from Decimal64.

base/base/extended_types.h

@@ -64,6 +64,44 @@ template <> struct is_arithmetic<UInt256> { static constexpr bool value = true;
 template <typename T>
 inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
 
+#define FOR_EACH_ARITHMETIC_TYPE(M) \
+    M(DataTypeDate) \
+    M(DataTypeDate32) \
+    M(DataTypeDateTime) \
+    M(DataTypeInt8) \
+    M(DataTypeUInt8) \
+    M(DataTypeInt16) \
+    M(DataTypeUInt16) \
+    M(DataTypeInt32) \
+    M(DataTypeUInt32) \
+    M(DataTypeInt64) \
+    M(DataTypeUInt64) \
+    M(DataTypeInt128) \
+    M(DataTypeUInt128) \
+    M(DataTypeInt256) \
+    M(DataTypeUInt256) \
+    M(DataTypeFloat32) \
+    M(DataTypeFloat64)
+
+#define FOR_EACH_ARITHMETIC_TYPE_PASS(M, X) \
+    M(DataTypeDate, X) \
+    M(DataTypeDate32, X) \
+    M(DataTypeDateTime, X) \
+    M(DataTypeInt8, X) \
+    M(DataTypeUInt8, X) \
+    M(DataTypeInt16, X) \
+    M(DataTypeUInt16, X) \
+    M(DataTypeInt32, X) \
+    M(DataTypeUInt32, X) \
+    M(DataTypeInt64, X) \
+    M(DataTypeUInt64, X) \
+    M(DataTypeInt128, X) \
+    M(DataTypeUInt128, X) \
+    M(DataTypeInt256, X) \
+    M(DataTypeUInt256, X) \
+    M(DataTypeFloat32, X) \
+    M(DataTypeFloat64, X)
+
 template <typename T>
 struct make_unsigned // NOLINT(readability-identifier-naming)
 {

docs/en/sql-reference/statements/grant.md

@@ -202,6 +202,13 @@ Hierarchy of privileges:
     - `S3`
 - [dictGet](#grant-dictget)
 - [displaySecretsInShowAndSelect](#grant-display-secrets)
+- [NAMED COLLECTION ADMIN](#grant-named-collection-admin)
+    - `CREATE NAMED COLLECTION`
+    - `DROP NAMED COLLECTION`
+    - `ALTER NAMED COLLECTION`
+    - `SHOW NAMED COLLECTIONS`
+    - `SHOW NAMED COLLECTIONS SECRETS`
+    - `NAMED COLLECTION`
 
 Examples of how this hierarchy is treated:
 
@@ -498,6 +505,25 @@ and
 [`format_display_secrets_in_show_and_select` format setting](../../operations/settings/formats#format_display_secrets_in_show_and_select)
 are turned on.
 
+### NAMED COLLECTION ADMIN
+
+Allows a certain operation on a specified named collection. Before version 23.7 it was called NAMED COLLECTION CONTROL, and after 23.7 NAMED COLLECTION ADMIN was added and NAMED COLLECTION CONTROL is preserved as an alias.
+
+- `NAMED COLLECTION ADMIN`. Level: `NAMED_COLLECTION`. Aliases: `NAMED COLLECTION CONTROL`
+    - `CREATE NAMED COLLECTION`. Level: `NAMED_COLLECTION`
+    - `DROP NAMED COLLECTION`. Level: `NAMED_COLLECTION`
+    - `ALTER NAMED COLLECTION`. Level: `NAMED_COLLECTION`
+    - `SHOW NAMED COLLECTIONS`. Level: `NAMED_COLLECTION`. Aliases: `SHOW NAMED COLLECTIONS`
+    - `SHOW NAMED COLLECTIONS SECRETS`. Level: `NAMED_COLLECTION`. Aliases: `SHOW NAMED COLLECTIONS SECRETS`
+    - `NAMED COLLECTION`. Level: `NAMED_COLLECTION`. Aliases: `NAMED COLLECTION USAGE, USE NAMED COLLECTION`
+
+Unlike all other grants (CREATE, DROP, ALTER, SHOW) grant NAMED COLLECTION was added only in 23.7, while all others were added earlier - in 22.12.
+
+**Examples**
+
+Assuming a named collection is called abc, we grant privilege CREATE NAMED COLLECTION to user john.
+- `GRANT CREATE NAMED COLLECTION ON abc TO john`
+
 ### ALL
 
 Grants all the privileges on regulated entity to a user account or a role.

programs/copier/Internals.cpp

@@ -259,7 +259,7 @@ ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std
     res.is_remote = 1;
     for (const auto & replica : replicas)
     {
-        if (isLocalAddress(DNSResolver::instance().resolveHost(replica.host_name)))
+        if (isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(replica.host_name).front()))
         {
             res.is_remote = 0;
             break;

src/Access/Common/AllowedClientHosts.cpp

@@ -55,7 +55,7 @@ namespace
     {
         IPAddress addr_v6 = toIPv6(address);
 
-        auto host_addresses = DNSResolver::instance().resolveHostAll(host);
+        auto host_addresses = DNSResolver::instance().resolveHostAllInOriginOrder(host);
 
         for (const auto & addr : host_addresses)
         {

src/Analyzer/ConstantNode.cpp

@@ -45,10 +45,15 @@ void ConstantNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state
     if (hasAlias())
         buffer << ", alias: " << getAlias();
 
-    buffer << ", constant_value: " << constant_value->getValue().dump();
+    buffer << ", constant_value: ";
+    if (mask_id)
+        buffer << "[HIDDEN id: " << mask_id << "]";
+    else
+        buffer << constant_value->getValue().dump();
+
     buffer << ", constant_value_type: " << constant_value->getType()->getName();
 
-    if (getSourceExpression())
+    if (!mask_id && getSourceExpression())
     {
         buffer << '\n' << std::string(indent + 2, ' ') << "EXPRESSION" << '\n';
         getSourceExpression()->dumpTreeImpl(buffer, format_state, indent + 4);

src/Analyzer/ConstantNode.h

@@ -75,6 +75,11 @@ public:
         return constant_value->getType();
     }
 
+    void setMaskId(size_t id)
+    {
+        mask_id = id;
+    }
+
     void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override;
 
 protected:
@@ -90,6 +95,7 @@ private:
     ConstantValuePtr constant_value;
     String value_string;
     QueryTreeNodePtr source_expression;
+    size_t mask_id = 0;
 
     static constexpr size_t children_size = 0;
 };

src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h

@@ -0,0 +1,372 @@
+#pragma once
+
+#include <Parsers/FunctionSecretArgumentsFinder.h>
+#include <Analyzer/ConstantNode.h>
+#include <Analyzer/FunctionNode.h>
+#include <Analyzer/IQueryTreeNode.h>
+#include <Analyzer/IdentifierNode.h>
+#include <Analyzer/ListNode.h>
+#include <Common/KnownObjectNames.h>
+#include <Core/QualifiedTableName.h>
+
+#include <boost/algorithm/string/predicate.hpp>
+
+
+namespace DB
+{
+
+
+/// Finds arguments of a specified function which should not be displayed for most users for security reasons.
+/// That involves passwords and secret keys.
+class FunctionSecretArgumentsFinderTreeNode
+{
+public:
+    explicit FunctionSecretArgumentsFinderTreeNode(const FunctionNode & function_) : function(function_), arguments(function.getArguments())
+    {
+        if (arguments.getNodes().empty())
+            return;
+
+        findFunctionSecretArguments();
+    }
+
+    struct Result
+    {
+        /// Result constructed by default means no arguments will be hidden.
+        size_t start = static_cast<size_t>(-1);
+        size_t count = 0; /// Mostly it's either 0 or 1. There are only a few cases where `count` can be greater than 1 (e.g. see `encrypt`).
+                            /// In all known cases secret arguments are consecutive
+        bool are_named = false; /// Arguments like `password = 'password'` are considered as named arguments.
+        /// E.g. "headers" in `url('..', headers('foo' = '[HIDDEN]'))`
+        std::vector<std::string> nested_maps;
+
+        bool hasSecrets() const
+        {
+            return count != 0 || !nested_maps.empty();
+        }
+    };
+
+    FunctionSecretArgumentsFinder::Result getResult() const { return result; }
+
+private:
+    const FunctionNode & function;
+    const ListNode & arguments;
+    FunctionSecretArgumentsFinder::Result result;
+
+    void markSecretArgument(size_t index, bool argument_is_named = false)
+    {
+        if (index >= arguments.getNodes().size())
+            return;
+        if (!result.count)
+        {
+            result.start = index;
+            result.are_named = argument_is_named;
+        }
+        chassert(index >= result.start); /// We always check arguments consecutively
+        result.count = index + 1 - result.start;
+        if (!argument_is_named)
+            result.are_named = false;
+    }
+
+    void findFunctionSecretArguments()
+    {
+        const auto & name = function.getFunctionName();
+
+        if ((name == "mysql") || (name == "postgresql") || (name == "mongodb"))
+        {
+            /// mysql('host:port', 'database', 'table', 'user', 'password', ...)
+            /// postgresql('host:port', 'database', 'table', 'user', 'password', ...)
+            /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...)
+            findMySQLFunctionSecretArguments();
+        }
+        else if ((name == "s3") || (name == "cosn") || (name == "oss") ||
+                    (name == "deltaLake") || (name == "hudi") || (name == "iceberg"))
+        {
+            /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
+            findS3FunctionSecretArguments(/* is_cluster_function= */ false);
+        }
+        else if (name == "s3Cluster")
+        {
+            /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...)
+            findS3FunctionSecretArguments(/* is_cluster_function= */ true);
+        }
+        else if ((name == "remote") || (name == "remoteSecure"))
+        {
+            /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...)
+            findRemoteFunctionSecretArguments();
+        }
+        else if ((name == "encrypt") || (name == "decrypt") ||
+                    (name == "aes_encrypt_mysql") || (name == "aes_decrypt_mysql") ||
+                    (name == "tryDecrypt"))
+        {
+            /// encrypt('mode', 'plaintext', 'key' [, iv, aad])
+            findEncryptionFunctionSecretArguments();
+        }
+        else if (name == "url")
+        {
+            findURLSecretArguments();
+        }
+    }
+
+    void findMySQLFunctionSecretArguments()
+    {
+        if (isNamedCollectionName(0))
+        {
+            /// mysql(named_collection, ..., password = 'password', ...)
+            findSecretNamedArgument("password", 1);
+        }
+        else
+        {
+            /// mysql('host:port', 'database', 'table', 'user', 'password', ...)
+            markSecretArgument(4);
+        }
+    }
+
+    /// Returns the number of arguments excluding "headers" and "extra_credentials" (which should
+    /// always be at the end). Marks "headers" as secret, if found.
+    size_t excludeS3OrURLNestedMaps()
+    {
+        const auto & nodes = arguments.getNodes();
+        size_t count = nodes.size();
+        while (count > 0)
+        {
+            const FunctionNode * f = nodes.at(count - 1)->as<FunctionNode>();
+            if (!f)
+                break;
+            if (f->getFunctionName() == "headers")
+                result.nested_maps.push_back(f->getFunctionName());
+            else if (f->getFunctionName() != "extra_credentials")
+                break;
+            count -= 1;
+        }
+        return count;
+    }
+
+    void findS3FunctionSecretArguments(bool is_cluster_function)
+    {
+        /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument.
+        size_t url_arg_idx = is_cluster_function ? 1 : 0;
+
+        if (!is_cluster_function && isNamedCollectionName(0))
+        {
+            /// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...)
+            findSecretNamedArgument("secret_access_key", 1);
+            return;
+        }
+
+        /// We should check other arguments first because we don't need to do any replacement in case of
+        /// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
+        /// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
+        size_t count = excludeS3OrURLNestedMaps();
+        if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4))
+        {
+            String second_arg;
+            if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg))
+            {
+                if (boost::iequals(second_arg, "NOSIGN"))
+                    return; /// The argument after 'url' is "NOSIGN".
+
+                if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg))
+                    return; /// The argument after 'url' is a format: s3('url', 'format', ...)
+            }
+        }
+
+        /// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures:
+        /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
+        /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
+        if (url_arg_idx + 2 < count)
+            markSecretArgument(url_arg_idx + 2);
+    }
+
+    void findURLSecretArguments()
+    {
+        if (!isNamedCollectionName(0))
+            excludeS3OrURLNestedMaps();
+    }
+
+    bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const
+    {
+        if (arg_idx >= arguments.getNodes().size())
+            return false;
+
+        return tryGetStringFromArgument(arguments.getNodes()[arg_idx], res, allow_identifier);
+    }
+
+    static bool tryGetStringFromArgument(const QueryTreeNodePtr argument, String * res, bool allow_identifier = true)
+    {
+        if (const auto * literal = argument->as<ConstantNode>())
+        {
+            if (literal->getValue().getType() != Field::Types::String)
+                return false;
+            if (res)
+                *res = literal->getValue().safeGet<String>();
+            return true;
+        }
+
+        if (allow_identifier)
+        {
+            if (const auto * id = argument->as<IdentifierNode>())
+            {
+                if (res)
+                    *res = id->getIdentifier().getFullName();
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    void findRemoteFunctionSecretArguments()
+    {
+        if (isNamedCollectionName(0))
+        {
+            /// remote(named_collection, ..., password = 'password', ...)
+            findSecretNamedArgument("password", 1);
+            return;
+        }
+
+        /// We're going to replace 'password' with '[HIDDEN'] for the following signatures:
+        /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key])
+        /// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key])
+        /// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key])
+
+        /// But we should check the number of arguments first because we don't need to do any replacements in case of
+        /// remote('addresses_expr', db.table)
+        if (arguments.getNodes().size() < 3)
+            return;
+
+        size_t arg_num = 1;
+
+        /// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'.
+        const auto * table_function = arguments.getNodes()[arg_num]->as<FunctionNode>();
+        if (table_function && KnownTableFunctionNames::instance().exists(table_function->getFunctionName()))
+        {
+            ++arg_num;
+        }
+        else
+        {
+            std::optional<String> database;
+            std::optional<QualifiedTableName> qualified_table_name;
+            if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name))
+            {
+                /// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'.
+                /// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user'
+                /// before the argument 'password'. So it's safer to wipe two arguments just in case.
+                /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
+                /// before wiping it (because the `password` argument is always a literal string).
+                if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false))
+                {
+                    /// Wipe either `password` or `user`.
+                    markSecretArgument(arg_num + 2);
+                }
+                if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false))
+                {
+                    /// Wipe either `password` or `sharding_key`.
+                    markSecretArgument(arg_num + 3);
+                }
+                return;
+            }
+
+            /// Skip the current argument (which is either a database name or a qualified table name).
+            ++arg_num;
+            if (database)
+            {
+                /// Skip the 'table' argument if the previous argument was a database name.
+                ++arg_num;
+            }
+        }
+
+        /// Skip username.
+        ++arg_num;
+
+        /// Do our replacement:
+        /// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...)
+        /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
+        /// before wiping it (because the `password` argument is always a literal string).
+        bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false);
+        if (can_be_password)
+            markSecretArgument(arg_num);
+    }
+
+    /// Tries to get either a database name or a qualified table name from an argument.
+    /// Empty string is also allowed (it means the default database).
+    /// The function is used by findRemoteFunctionSecretArguments() to determine how many arguments to skip before a password.
+    bool tryGetDatabaseNameOrQualifiedTableName(
+        size_t arg_idx,
+        std::optional<String> & res_database,
+        std::optional<QualifiedTableName> & res_qualified_table_name) const
+    {
+        res_database.reset();
+        res_qualified_table_name.reset();
+
+        String str;
+        if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true))
+            return false;
+
+        if (str.empty())
+        {
+            res_database = "";
+            return true;
+        }
+
+        auto qualified_table_name = QualifiedTableName::tryParseFromString(str);
+        if (!qualified_table_name)
+            return false;
+
+        if (qualified_table_name->database.empty())
+            res_database = std::move(qualified_table_name->table);
+        else
+            res_qualified_table_name = std::move(qualified_table_name);
+        return true;
+    }
+
+    void findEncryptionFunctionSecretArguments()
+    {
+        if (arguments.getNodes().empty())
+            return;
+
+        /// We replace all arguments after 'mode' with '[HIDDEN]':
+        /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]')
+        result.start = 1;
+        result.count = arguments.getNodes().size() - 1;
+    }
+
+
+    /// Whether a specified argument can be the name of a named collection?
+    bool isNamedCollectionName(size_t arg_idx) const
+    {
+        if (arguments.getNodes().size() <= arg_idx)
+            return false;
+
+        const auto * identifier = arguments.getNodes()[arg_idx]->as<IdentifierNode>();
+        return identifier != nullptr;
+    }
+
+    /// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified.
+    void findSecretNamedArgument(const std::string_view & key, size_t start = 0)
+    {
+        for (size_t i = start; i < arguments.getNodes().size(); ++i)
+        {
+            const auto & argument = arguments.getNodes()[i];
+            const auto * equals_func = argument->as<FunctionNode>();
+            if (!equals_func || (equals_func->getFunctionName() != "equals"))
+                continue;
+
+            const auto * expr_list = equals_func->getArguments().as<ListNode>();
+            if (!expr_list)
+                continue;
+
+            const auto & equal_args = expr_list->getNodes();
+            if (equal_args.size() != 2)
+                continue;
+
+            String found_key;
+            if (!tryGetStringFromArgument(equal_args[0], &found_key))
+                continue;
+
+            if (found_key == key)
+                markSecretArgument(i, /* argument_is_named= */ true);
+        }
+    }
+};
+
+}

src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp

@@ -94,7 +94,8 @@ public:
         if (!func_node || func_node->getArguments().getNodes().size() != 1)
             return;
 
-        const auto * column_id = func_node->getArguments().getNodes()[0]->as<ColumnNode>();
+        const auto & argument_node = func_node->getArguments().getNodes()[0];
+        const auto * column_id = argument_node->as<ColumnNode>();
         if (!column_id)
             return;
 
@@ -119,7 +120,7 @@ public:
         if (!preimage_range)
             return;
 
-        const auto new_node = generateOptimizedDateFilter(comparator, *column_id, *preimage_range);
+        const auto new_node = generateOptimizedDateFilter(comparator, argument_node, *preimage_range);
 
         if (!new_node)
             return;
@@ -128,20 +129,22 @@ public:
     }
 
 private:
-    QueryTreeNodePtr
-    generateOptimizedDateFilter(const String & comparator, const ColumnNode & column_node, const std::pair<Field, Field> & range) const
+    QueryTreeNodePtr generateOptimizedDateFilter(
+        const String & comparator, const QueryTreeNodePtr & column_node, const std::pair<Field, Field> & range) const
     {
         const DateLUTImpl & date_lut = DateLUT::instance("UTC");
 
         String start_date_or_date_time;
         String end_date_or_date_time;
 
-        if (isDateOrDate32(column_node.getColumnType().get()))
+        const auto & column_node_typed = column_node->as<ColumnNode &>();
+        const auto & column_type = column_node_typed.getColumnType().get();
+        if (isDateOrDate32(column_type))
         {
             start_date_or_date_time = date_lut.dateToString(range.first.get<DateLUTImpl::Time>());
             end_date_or_date_time = date_lut.dateToString(range.second.get<DateLUTImpl::Time>());
         }
-        else if (isDateTime(column_node.getColumnType().get()) || isDateTime64(column_node.getColumnType().get()))
+        else if (isDateTime(column_type) || isDateTime64(column_type))
         {
             start_date_or_date_time = date_lut.timeToString(range.first.get<DateLUTImpl::Time>());
             end_date_or_date_time = date_lut.timeToString(range.second.get<DateLUTImpl::Time>());
@@ -151,69 +154,29 @@ private:
 
         if (comparator == "equals")
         {
-            const auto lhs = std::make_shared<FunctionNode>("greaterOrEquals");
-            lhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            lhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(start_date_or_date_time));
-            resolveOrdinaryFunctionNode(*lhs, lhs->getFunctionName());
-
-            const auto rhs = std::make_shared<FunctionNode>("less");
-            rhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            rhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
-            resolveOrdinaryFunctionNode(*rhs, rhs->getFunctionName());
-
-            const auto new_date_filter = std::make_shared<FunctionNode>("and");
-            new_date_filter->getArguments().getNodes() = {lhs, rhs};
-            resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
-
-            return new_date_filter;
+            return createFunctionNode(
+                "and",
+                createFunctionNode("greaterOrEquals", column_node, std::make_shared<ConstantNode>(start_date_or_date_time)),
+                createFunctionNode("less", column_node, std::make_shared<ConstantNode>(end_date_or_date_time)));
         }
         else if (comparator == "notEquals")
         {
-            const auto lhs = std::make_shared<FunctionNode>("less");
-            lhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            lhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(start_date_or_date_time));
-            resolveOrdinaryFunctionNode(*lhs, lhs->getFunctionName());
-
-            const auto rhs = std::make_shared<FunctionNode>("greaterOrEquals");
-            rhs->getArguments().getNodes().push_back(std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            rhs->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
-            resolveOrdinaryFunctionNode(*rhs, rhs->getFunctionName());
-
-            const auto new_date_filter = std::make_shared<FunctionNode>("or");
-            new_date_filter->getArguments().getNodes() = {lhs, rhs};
-            resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
-
-            return new_date_filter;
+            return createFunctionNode(
+                "or",
+                createFunctionNode("less", column_node, std::make_shared<ConstantNode>(start_date_or_date_time)),
+                createFunctionNode("greaterOrEquals", column_node, std::make_shared<ConstantNode>(end_date_or_date_time)));
         }
         else if (comparator == "greater")
         {
-            const auto new_date_filter = std::make_shared<FunctionNode>("greaterOrEquals");
-            new_date_filter->getArguments().getNodes().push_back(
-                std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            new_date_filter->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
-            resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
-
-            return new_date_filter;
+            return createFunctionNode("greaterOrEquals", column_node, std::make_shared<ConstantNode>(end_date_or_date_time));
         }
         else if (comparator == "lessOrEquals")
         {
-            const auto new_date_filter = std::make_shared<FunctionNode>("less");
-            new_date_filter->getArguments().getNodes().push_back(
-                std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            new_date_filter->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(end_date_or_date_time));
-            resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
-
-            return new_date_filter;
+            return createFunctionNode("less", column_node, std::make_shared<ConstantNode>(end_date_or_date_time));
         }
         else if (comparator == "less" || comparator == "greaterOrEquals")
         {
-            const auto new_date_filter = std::make_shared<FunctionNode>(comparator);
-            new_date_filter->getArguments().getNodes().push_back(
-                std::make_shared<ColumnNode>(column_node.getColumn(), column_node.getColumnSource()));
-            new_date_filter->getArguments().getNodes().push_back(std::make_shared<ConstantNode>(start_date_or_date_time));
-            resolveOrdinaryFunctionNode(*new_date_filter, new_date_filter->getFunctionName());
-
-            return new_date_filter;
+            return createFunctionNode(comparator, column_node, std::make_shared<ConstantNode>(start_date_or_date_time));
         }
         else [[unlikely]]
         {
@@ -224,10 +187,17 @@ private:
         }
     }
 
-    void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const
+    template <typename... Args>
+    QueryTreeNodePtr createFunctionNode(const String & function_name, Args &&... args) const
     {
         auto function = FunctionFactory::instance().get(function_name, getContext());
-        function_node.resolveAsFunction(function->build(function_node.getArgumentColumns()));
+        const auto function_node = std::make_shared<FunctionNode>(function_name);
+        auto & new_arguments = function_node->getArguments().getNodes();
+        new_arguments.reserve(sizeof...(args));
+        (new_arguments.push_back(std::forward<Args>(args)), ...);
+        function_node->resolveAsFunction(function->build(function_node->getArgumentColumns()));
+
+        return function_node;
     }
 };
 

src/Analyzer/Passes/QueryAnalysisPass.cpp

@@ -3,6 +3,7 @@
 #include <Common/checkStackSize.h>
 #include <Common/NamePrompter.h>
 #include <Common/ProfileEvents.h>
+#include <Analyzer/FunctionSecretArgumentsFinderTreeNode.h>
 
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
@@ -706,7 +707,10 @@ struct IdentifierResolveScope
         {
             subquery_depth = parent_scope->subquery_depth;
             context = parent_scope->context;
+            projection_mask_map = parent_scope->projection_mask_map;
         }
+        else
+            projection_mask_map = std::make_shared<std::map<IQueryTreeNode::Hash, size_t>>();
 
         if (auto * union_node = scope_node->as<UnionNode>())
         {
@@ -718,6 +722,11 @@ struct IdentifierResolveScope
             group_by_use_nulls = context->getSettingsRef().group_by_use_nulls &&
                 (query_node->isGroupByWithGroupingSets() || query_node->isGroupByWithRollup() || query_node->isGroupByWithCube());
         }
+
+        if (context)
+            join_use_nulls = context->getSettingsRef().join_use_nulls;
+        else if (parent_scope)
+            join_use_nulls = parent_scope->join_use_nulls;
     }
 
     QueryTreeNodePtr scope_node;
@@ -772,6 +781,8 @@ struct IdentifierResolveScope
 
     /// Apply nullability to aggregation keys
     bool group_by_use_nulls = false;
+    /// Join retutns NULLs instead of default values
+    bool join_use_nulls = false;
 
     /// JOINs count
     size_t joins_count = 0;
@@ -784,6 +795,9 @@ struct IdentifierResolveScope
       */
     QueryTreeNodePtr expression_join_tree_node;
 
+    /// Node hash to mask id map
+    std::shared_ptr<std::map<IQueryTreeNode::Hash, size_t>> projection_mask_map;
+
     [[maybe_unused]] const IdentifierResolveScope * getNearestQueryScope() const
     {
         const IdentifierResolveScope * scope_to_check = this;
@@ -3286,7 +3300,6 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
     QueryTreeNodePtr resolved_identifier;
 
     JoinKind join_kind = from_join_node.getKind();
-    bool join_use_nulls = scope.context->getSettingsRef().join_use_nulls;
 
     /// If columns from left or right table were missed Object(Nullable('json')) subcolumns, they will be replaced
     /// to ConstantNode(NULL), which can't be cast to ColumnNode, so we resolve it here.
@@ -3451,7 +3464,7 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
     if (join_node_in_resolve_process || !resolved_identifier)
         return resolved_identifier;
 
-    if (join_use_nulls)
+    if (scope.join_use_nulls)
     {
         resolved_identifier = resolved_identifier->clone();
         convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side);
@@ -4439,7 +4452,7 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I
     else
         matched_expression_nodes_with_names = resolveUnqualifiedMatcher(matcher_node, scope);
 
-    if (scope.context->getSettingsRef().join_use_nulls)
+    if (scope.join_use_nulls)
     {
         /** If we are resolving matcher came from the result of JOIN and `join_use_nulls` is set,
           * we need to convert joined column type to Nullable.
@@ -5124,22 +5137,31 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
     }
 
     /// Resolve function arguments
-
     bool allow_table_expressions = is_special_function_in;
     auto arguments_projection_names = resolveExpressionNodeList(function_node_ptr->getArgumentsNode(),
         scope,
         true /*allow_lambda_expression*/,
         allow_table_expressions /*allow_table_expression*/);
 
-    if (function_node_ptr->toAST()->hasSecretParts())
+    /// Mask arguments if needed
+    if (!scope.context->getSettingsRef().format_display_secrets_in_show_and_select)
     {
-        for (auto & argument : arguments_projection_names)
+        if (FunctionSecretArgumentsFinder::Result secret_arguments = FunctionSecretArgumentsFinderTreeNode(*function_node_ptr).getResult(); secret_arguments.count)
         {
-            SipHash hash;
-            hash.update(argument);
-            argument = getHexUIntLowercase(hash.get128());
+            auto & argument_nodes = function_node_ptr->getArgumentsNode()->as<ListNode &>().getNodes();
+
+            for (size_t n = secret_arguments.start; n < secret_arguments.start + secret_arguments.count; ++n)
+            {
+                if (auto * constant = argument_nodes[n]->as<ConstantNode>())
+                {
+                    auto mask = scope.projection_mask_map->insert({constant->getTreeHash(), scope.projection_mask_map->size() + 1}).first->second;
+                    constant->setMaskId(mask);
+                    arguments_projection_names[n] = "[HIDDEN id: " + std::to_string(mask) + "]";
+                }
+            }
         }
     }
+
     auto & function_node = *function_node_ptr;
 
     /// Replace right IN function argument if it is table or table function with subquery that read ordinary columns
@@ -7559,8 +7581,22 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
     }
 
     if (query_node_typed.getPrewhere())
+    {
+        /** Expression in PREWHERE with JOIN should not be modified by join_use_nulls.
+          * Example: SELECT * FROM t1 JOIN t2 USING (id) PREWHERE a = 1
+          * Column `a` should be resolved from table and should not change its type to Nullable.
+          */
+        bool join_use_nulls = scope.join_use_nulls;
+        bool use_identifier_lookup_to_result_cache = scope.use_identifier_lookup_to_result_cache;
+        scope.join_use_nulls = false;
+        scope.use_identifier_lookup_to_result_cache = false;
+
         resolveExpressionNode(query_node_typed.getPrewhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
 
+        scope.join_use_nulls = join_use_nulls;
+        scope.use_identifier_lookup_to_result_cache = use_identifier_lookup_to_result_cache;
+    }
+
     if (query_node_typed.getWhere())
         resolveExpressionNode(query_node_typed.getWhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
 

src/Client/ConnectionParameters.cpp

@@ -115,7 +115,7 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
     /// At the same time, I want clickhouse-local to always work, regardless.
     /// TODO: get rid of glibc, or replace getaddrinfo to c-ares.
 
-    compression = config.getBool("compression", host != "localhost" && !isLocalAddress(DNSResolver::instance().resolveHost(host)))
+    compression = config.getBool("compression", host != "localhost" && !isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(host).front()))
                   ? Protocol::Compression::Enable : Protocol::Compression::Disable;
 
     timeouts = ConnectionTimeouts()

src/Columns/ColumnUnique.cpp

@@ -0,0 +1,25 @@
+#include <Columns/ColumnUnique.h>
+
+namespace DB
+{
+
+/// Explicit template instantiations.
+template class ColumnUnique<ColumnInt8>;
+template class ColumnUnique<ColumnUInt8>;
+template class ColumnUnique<ColumnInt16>;
+template class ColumnUnique<ColumnUInt16>;
+template class ColumnUnique<ColumnInt32>;
+template class ColumnUnique<ColumnUInt32>;
+template class ColumnUnique<ColumnInt64>;
+template class ColumnUnique<ColumnUInt64>;
+template class ColumnUnique<ColumnInt128>;
+template class ColumnUnique<ColumnUInt128>;
+template class ColumnUnique<ColumnInt256>;
+template class ColumnUnique<ColumnUInt256>;
+template class ColumnUnique<ColumnFloat32>;
+template class ColumnUnique<ColumnFloat64>;
+template class ColumnUnique<ColumnString>;
+template class ColumnUnique<ColumnFixedString>;
+template class ColumnUnique<ColumnDateTime64>;
+
+}

src/Columns/ColumnUnique.h

@@ -15,6 +15,8 @@
 #include <Common/typeid_cast.h>
 #include <Common/assert_cast.h>
 #include <Common/FieldVisitors.h>
+#include "Columns/ColumnsDateTime.h"
+#include "Columns/ColumnsNumber.h"
 
 #include <base/range.h>
 #include <base/unaligned.h>
@@ -736,4 +738,23 @@ UInt128 ColumnUnique<ColumnType>::IncrementalHash::getHash(const ColumnType & co
     return cur_hash;
 }
 
+
+extern template class ColumnUnique<ColumnInt8>;
+extern template class ColumnUnique<ColumnUInt8>;
+extern template class ColumnUnique<ColumnInt16>;
+extern template class ColumnUnique<ColumnUInt16>;
+extern template class ColumnUnique<ColumnInt32>;
+extern template class ColumnUnique<ColumnUInt32>;
+extern template class ColumnUnique<ColumnInt64>;
+extern template class ColumnUnique<ColumnUInt64>;
+extern template class ColumnUnique<ColumnInt128>;
+extern template class ColumnUnique<ColumnUInt128>;
+extern template class ColumnUnique<ColumnInt256>;
+extern template class ColumnUnique<ColumnUInt256>;
+extern template class ColumnUnique<ColumnFloat32>;
+extern template class ColumnUnique<ColumnFloat64>;
+extern template class ColumnUnique<ColumnString>;
+extern template class ColumnUnique<ColumnFixedString>;
+extern template class ColumnUnique<ColumnDateTime64>;
+
 }

src/Common/DNSResolver.cpp

@@ -202,10 +202,10 @@ DNSResolver::DNSResolver() : impl(std::make_unique<DNSResolver::Impl>()), log(ge
 
 Poco::Net::IPAddress DNSResolver::resolveHost(const std::string & host)
 {
-    return pickAddress(resolveHostAll(host));
+    return pickAddress(resolveHostAll(host)); // random order -> random pick
 }
 
-DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
+DNSResolver::IPAddresses DNSResolver::resolveHostAllInOriginOrder(const std::string & host)
 {
     if (impl->disable_cache)
         return resolveIPAddressImpl(host);
@@ -214,6 +214,13 @@ DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
     return resolveIPAddressWithCache(impl->cache_host, host);
 }
 
+DNSResolver::IPAddresses DNSResolver::resolveHostAll(const std::string & host)
+{
+    auto addresses = resolveHostAllInOriginOrder(host);
+    std::shuffle(addresses.begin(), addresses.end(), thread_local_rng);
+    return addresses;
+}
+
 Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host_and_port)
 {
     if (impl->disable_cache)

src/Common/DNSResolver.h

@@ -34,6 +34,9 @@ public:
     Poco::Net::IPAddress resolveHost(const std::string & host);
 
     /// Accepts host names like 'example.com' or '127.0.0.1' or '::1' and resolves all its IPs
+    /// resolveHostAllInOriginOrder returns addresses with the same order as system call returns it
+    IPAddresses resolveHostAllInOriginOrder(const std::string & host);
+    /// resolveHostAll returns addresses in random order
     IPAddresses resolveHostAll(const std::string & host);
 
     /// Accepts host names like 'example.com:port' or '127.0.0.1:port' or '[::1]:port' and resolves its IP and port

src/Common/FieldVisitorConvertToNumber.cpp

@@ -0,0 +1,23 @@
+#include <Common/FieldVisitorConvertToNumber.h>
+#include "base/Decimal.h"
+
+namespace DB
+{
+
+/// Explicit template instantiations.
+template class FieldVisitorConvertToNumber<Int8>;
+template class FieldVisitorConvertToNumber<UInt8>;
+template class FieldVisitorConvertToNumber<Int16>;
+template class FieldVisitorConvertToNumber<UInt16>;
+template class FieldVisitorConvertToNumber<Int32>;
+template class FieldVisitorConvertToNumber<UInt32>;
+template class FieldVisitorConvertToNumber<Int64>;
+template class FieldVisitorConvertToNumber<UInt64>;
+template class FieldVisitorConvertToNumber<Int128>;
+template class FieldVisitorConvertToNumber<UInt128>;
+template class FieldVisitorConvertToNumber<Int256>;
+template class FieldVisitorConvertToNumber<UInt256>;
+template class FieldVisitorConvertToNumber<Float32>;
+template class FieldVisitorConvertToNumber<Float64>;
+
+}

src/Common/FieldVisitorConvertToNumber.h

@@ -117,4 +117,19 @@ public:
     T operator() (const bool & x) const { return T(x); }
 };
 
+extern template class FieldVisitorConvertToNumber<Int8>;
+extern template class FieldVisitorConvertToNumber<UInt8>;
+extern template class FieldVisitorConvertToNumber<Int16>;
+extern template class FieldVisitorConvertToNumber<UInt16>;
+extern template class FieldVisitorConvertToNumber<Int32>;
+extern template class FieldVisitorConvertToNumber<UInt32>;
+extern template class FieldVisitorConvertToNumber<Int64>;
+extern template class FieldVisitorConvertToNumber<UInt64>;
+extern template class FieldVisitorConvertToNumber<Int128>;
+extern template class FieldVisitorConvertToNumber<UInt128>;
+extern template class FieldVisitorConvertToNumber<Int256>;
+extern template class FieldVisitorConvertToNumber<UInt256>;
+extern template class FieldVisitorConvertToNumber<Float32>;
+extern template class FieldVisitorConvertToNumber<Float64>;
+
 }

src/Coordination/KeeperStateManager.cpp

@@ -30,7 +30,7 @@ bool isLocalhost(const std::string & hostname)
 {
     try
     {
-        return isLocalAddress(DNSResolver::instance().resolveHost(hostname));
+        return isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(hostname).front());
     }
     catch (...)
     {

src/Core/Settings.h

@@ -186,6 +186,7 @@ class IColumn;
     \
     M(String, cluster_for_parallel_replicas, "", "Cluster for a shard in which current server is located", 0) \
     M(UInt64, allow_experimental_parallel_reading_from_replicas, 0, "Use all the replicas from a shard for SELECT query execution. Reading is parallelized and coordinated dynamically. 0 - disabled, 1 - enabled, silently disable them in case of failure, 2 - enabled, throw an exception in case of failure", 0) \
+    M(Bool, parallel_replicas_allow_in_with_subquery, true, "If true, subquery for IN will be executed on every follower replica.", 0) \
     M(Float, parallel_replicas_single_task_marks_count_multiplier, 2, "A multiplier which will be added during calculation for minimal number of marks to retrieve from coordinator. This will be applied only for remote replicas.", 0) \
     M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \
     M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \

src/Core/SettingsChangesHistory.h

@@ -90,6 +90,7 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
               {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
               {"page_cache_inject_eviction", false, false, "Added userspace page cache"},
               {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
+              {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
               }},
     {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
               {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},

src/DataTypes/DataTypeDecimalBase.h

@@ -207,4 +207,10 @@ inline DataTypePtr createDecimal(UInt64 precision_value, UInt64 scale_value)
     return std::make_shared<DecimalType<Decimal256>>(precision_value, scale_value);
 }
 
+extern template class DataTypeDecimalBase<Decimal32>;
+extern template class DataTypeDecimalBase<Decimal64>;
+extern template class DataTypeDecimalBase<Decimal128>;
+extern template class DataTypeDecimalBase<Decimal256>;
+extern template class DataTypeDecimalBase<DateTime64>;
+
 }

src/DataTypes/DataTypesDecimal.cpp

@@ -112,6 +112,256 @@ static DataTypePtr createExact(const ASTPtr & arguments)
     return createDecimal<DataTypeDecimal>(precision, scale);
 }
 
+template <typename FromDataType, typename ToDataType, typename ReturnType>
+requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
+ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
+{
+    using FromFieldType = typename FromDataType::FieldType;
+    using ToFieldType = typename ToDataType::FieldType;
+    using MaxFieldType = std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), FromFieldType, ToFieldType>;
+    using MaxNativeType = typename MaxFieldType::NativeType;
+
+    static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
+
+    MaxNativeType converted_value;
+    if (scale_to > scale_from)
+    {
+        converted_value = DecimalUtils::scaleMultiplier<MaxNativeType>(scale_to - scale_from);
+        if (common::mulOverflow(static_cast<MaxNativeType>(value.value), converted_value, converted_value))
+        {
+            if constexpr (throw_exception)
+                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow while multiplying {} by scale {}",
+                                std::string(ToDataType::family_name), toString(value.value), toString(converted_value));
+            else
+                return ReturnType(false);
+        }
+    }
+    else if (scale_to == scale_from)
+    {
+        converted_value = value.value;
+    }
+    else
+    {
+        converted_value = value.value / DecimalUtils::scaleMultiplier<MaxNativeType>(scale_from - scale_to);
+    }
+
+    if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType))
+    {
+        if (converted_value < std::numeric_limits<typename ToFieldType::NativeType>::min() ||
+            converted_value > std::numeric_limits<typename ToFieldType::NativeType>::max())
+        {
+            if constexpr (throw_exception)
+                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow: {} is not in range ({}, {})",
+                                std::string(ToDataType::family_name), toString(converted_value),
+                                toString(std::numeric_limits<typename ToFieldType::NativeType>::min()),
+                                toString(std::numeric_limits<typename ToFieldType::NativeType>::max()));
+            else
+                return ReturnType(false);
+        }
+    }
+
+    result = static_cast<typename ToFieldType::NativeType>(converted_value);
+
+    return ReturnType(true);
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template void convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, void>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result); \
+    template bool convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, bool>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
+typename ToDataType::FieldType convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to)
+{
+    using ToFieldType = typename ToDataType::FieldType;
+    ToFieldType result;
+
+    convertDecimalsImpl<FromDataType, ToDataType, void>(value, scale_from, scale_to, result);
+
+    return result;
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template typename TO_DATA_TYPE::FieldType convertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
+bool tryConvertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
+{
+    return convertDecimalsImpl<FromDataType, ToDataType, bool>(value, scale_from, scale_to, result);
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template bool tryConvertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType, typename ReturnType>
+requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
+ReturnType convertFromDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType & result)
+{
+    using FromFieldType = typename FromDataType::FieldType;
+    using ToFieldType = typename ToDataType::FieldType;
+
+    return DecimalUtils::convertToImpl<ToFieldType, FromFieldType, ReturnType>(value, scale, result);
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template void convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result); \
+    template bool convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
+inline typename ToDataType::FieldType convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
+{
+    typename ToDataType::FieldType result;
+
+    convertFromDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
+
+    return result;
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template typename TO_DATA_TYPE::FieldType convertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
+inline bool tryConvertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
+{
+    return convertFromDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template bool tryConvertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType, typename ReturnType>
+requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
+ReturnType convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType & result)
+{
+    using FromFieldType = typename FromDataType::FieldType;
+    using ToFieldType = typename ToDataType::FieldType;
+    using ToNativeType = typename ToFieldType::NativeType;
+
+    static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
+
+    if constexpr (std::is_floating_point_v<FromFieldType>)
+    {
+        if (!std::isfinite(value))
+        {
+            if constexpr (throw_exception)
+                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Cannot convert infinity or NaN to decimal", ToDataType::family_name);
+            else
+                return ReturnType(false);
+        }
+
+        auto out = value * static_cast<FromFieldType>(DecimalUtils::scaleMultiplier<ToNativeType>(scale));
+
+        if (out <= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::min()) ||
+            out >= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::max()))
+        {
+            if constexpr (throw_exception)
+                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Float is out of Decimal range", ToDataType::family_name);
+            else
+                return ReturnType(false);
+        }
+
+        result = static_cast<ToNativeType>(out);
+        return ReturnType(true);
+    }
+    else
+    {
+        if constexpr (is_big_int_v<FromFieldType>)
+            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal256>, ToDataType, ReturnType>(static_cast<Int256>(value), 0, scale, result));
+        else if constexpr (std::is_same_v<FromFieldType, UInt64>)
+            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal128>, ToDataType, ReturnType>(static_cast<Int128>(value), 0, scale, result));
+        else
+            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal64>, ToDataType, ReturnType>(static_cast<Int64>(value), 0, scale, result));
+    }
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template void convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);  \
+    template bool convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
+inline typename ToDataType::FieldType convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
+{
+    typename ToDataType::FieldType result;
+    convertToDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
+    return result;
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template typename TO_DATA_TYPE::FieldType convertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename FromDataType, typename ToDataType>
+requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
+inline bool tryConvertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
+{
+    return convertToDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
+}
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    template bool tryConvertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
+
+template <typename T>
+DataTypePtr createDecimalMaxPrecision(UInt64 scale)
+{
+    return std::make_shared<DataTypeDecimal<T>>(DecimalUtils::max_precision<T>, scale);
+}
+
+template DataTypePtr createDecimalMaxPrecision<Decimal32>(UInt64 scale);
+template DataTypePtr createDecimalMaxPrecision<Decimal64>(UInt64 scale);
+template DataTypePtr createDecimalMaxPrecision<Decimal128>(UInt64 scale);
+template DataTypePtr createDecimalMaxPrecision<Decimal256>(UInt64 scale);
+
+/// Explicit template instantiations.
+template class DataTypeDecimal<Decimal32>;
+template class DataTypeDecimal<Decimal64>;
+template class DataTypeDecimal<Decimal128>;
+template class DataTypeDecimal<Decimal256>;
+
 void registerDataTypeDecimal(DataTypeFactory & factory)
 {
     factory.registerDataType("Decimal32", createExact<Decimal32>, DataTypeFactory::CaseInsensitive);
@@ -125,10 +375,4 @@ void registerDataTypeDecimal(DataTypeFactory & factory)
     factory.registerAlias("FIXED", "Decimal", DataTypeFactory::CaseInsensitive);
 }
 
-/// Explicit template instantiations.
-template class DataTypeDecimal<Decimal32>;
-template class DataTypeDecimal<Decimal64>;
-template class DataTypeDecimal<Decimal128>;
-template class DataTypeDecimal<Decimal256>;
-
 }

src/DataTypes/DataTypesDecimal.h

@@ -3,7 +3,11 @@
 #include <base/arithmeticOverflow.h>
 #include <base/extended_types.h>
 #include <Common/typeid_cast.h>
+#include <base/Decimal.h>
+#include <base/Decimal_fwd.h>
 #include <DataTypes/IDataType.h>
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDecimalBase.h>
 #include <DataTypes/DataTypeDateTime64.h>
 
@@ -13,7 +17,6 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int DECIMAL_OVERFLOW;
     extern const int LOGICAL_ERROR;
 }
 
@@ -99,171 +102,145 @@ inline UInt32 getDecimalScale(const DataTypeDecimal<T> & data_type)
     return data_type.getScale();
 }
 
+#define FOR_EACH_DECIMAL_TYPE(M) \
+    M(DataTypeDecimal<DateTime64>) \
+    M(DataTypeDateTime64) \
+    M(DataTypeDecimal32) \
+    M(DataTypeDecimal64) \
+    M(DataTypeDecimal128) \
+    M(DataTypeDecimal256)
+
+#define FOR_EACH_DECIMAL_TYPE_PASS(M, X) \
+    M(DataTypeDecimal<DateTime64>, X) \
+    M(DataTypeDateTime64, X) \
+    M(DataTypeDecimal32, X) \
+    M(DataTypeDecimal64, X) \
+    M(DataTypeDecimal128, X) \
+    M(DataTypeDecimal256, X)
+
+
 template <typename FromDataType, typename ToDataType, typename ReturnType = void>
 requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
-inline ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
-{
-    using FromFieldType = typename FromDataType::FieldType;
-    using ToFieldType = typename ToDataType::FieldType;
-    using MaxFieldType = std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), FromFieldType, ToFieldType>;
-    using MaxNativeType = typename MaxFieldType::NativeType;
+ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result);
 
-    static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template void convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, void>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result); \
+    extern template bool convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, bool>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
 
-    MaxNativeType converted_value;
-    if (scale_to > scale_from)
-    {
-        converted_value = DecimalUtils::scaleMultiplier<MaxNativeType>(scale_to - scale_from);
-        if (common::mulOverflow(static_cast<MaxNativeType>(value.value), converted_value, converted_value))
-        {
-            if constexpr (throw_exception)
-                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow while multiplying {} by scale {}",
-                                std::string(ToDataType::family_name), toString(value.value), toString(converted_value));
-            else
-                return ReturnType(false);
-        }
-    }
-    else if (scale_to == scale_from)
-    {
-        converted_value = value.value;
-    }
-    else
-    {
-        converted_value = value.value / DecimalUtils::scaleMultiplier<MaxNativeType>(scale_from - scale_to);
-    }
-
-    if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType))
-    {
-        if (converted_value < std::numeric_limits<typename ToFieldType::NativeType>::min() ||
-            converted_value > std::numeric_limits<typename ToFieldType::NativeType>::max())
-        {
-            if constexpr (throw_exception)
-                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow: {} is not in range ({}, {})",
-                                std::string(ToDataType::family_name), toString(converted_value),
-                                toString(std::numeric_limits<typename ToFieldType::NativeType>::min()),
-                                toString(std::numeric_limits<typename ToFieldType::NativeType>::max()));
-            else
-                return ReturnType(false);
-        }
-    }
-
-    result = static_cast<typename ToFieldType::NativeType>(converted_value);
-
-    return ReturnType(true);
-}
 
 template <typename FromDataType, typename ToDataType>
 requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
-inline typename ToDataType::FieldType convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to)
-{
-    using ToFieldType = typename ToDataType::FieldType;
-    ToFieldType result;
+typename ToDataType::FieldType convertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to);
 
-    convertDecimalsImpl<FromDataType, ToDataType, void>(value, scale_from, scale_to, result);
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template typename TO_DATA_TYPE::FieldType convertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
 
-    return result;
-}
 
 template <typename FromDataType, typename ToDataType>
 requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
-inline bool tryConvertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result)
-{
-    return convertDecimalsImpl<FromDataType, ToDataType, bool>(value, scale_from, scale_to, result);
-}
+bool tryConvertDecimals(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result);
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template bool tryConvertDecimals<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
 
 template <typename FromDataType, typename ToDataType, typename ReturnType>
 requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
-inline ReturnType convertFromDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
-{
-    using FromFieldType = typename FromDataType::FieldType;
-    using ToFieldType = typename ToDataType::FieldType;
+ReturnType convertFromDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType & result);
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template void convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result); \
+    extern template bool convertFromDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
 
-    return DecimalUtils::convertToImpl<ToFieldType, FromFieldType, ReturnType>(value, scale, result);
-}
 
 template <typename FromDataType, typename ToDataType>
 requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
-inline typename ToDataType::FieldType convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
-{
-    typename ToDataType::FieldType result;
+typename ToDataType::FieldType convertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale);
 
-    convertFromDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template typename TO_DATA_TYPE::FieldType convertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
 
-    return result;
-}
 
 template <typename FromDataType, typename ToDataType>
 requires (IsDataTypeDecimal<FromDataType> && is_arithmetic_v<typename ToDataType::FieldType>)
-inline bool tryConvertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
-{
-    return convertFromDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
-}
+bool tryConvertFromDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result);
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template bool tryConvertFromDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
+#define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
+FOR_EACH_ARITHMETIC_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
 
 template <typename FromDataType, typename ToDataType, typename ReturnType>
 requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
-inline ReturnType convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
-{
-    using FromFieldType = typename FromDataType::FieldType;
-    using ToFieldType = typename ToDataType::FieldType;
-    using ToNativeType = typename ToFieldType::NativeType;
+ReturnType convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result);
 
-    static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template void convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);  \
+    extern template bool convertToDecimalImpl<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType & result);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
 
-    if constexpr (std::is_floating_point_v<FromFieldType>)
-    {
-        if (!std::isfinite(value))
-        {
-            if constexpr (throw_exception)
-                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Cannot convert infinity or NaN to decimal", ToDataType::family_name);
-            else
-                return ReturnType(false);
-        }
-
-        auto out = value * static_cast<FromFieldType>(DecimalUtils::scaleMultiplier<ToNativeType>(scale));
-
-        if (out <= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::min()) ||
-            out >= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::max()))
-        {
-            if constexpr (throw_exception)
-                throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Float is out of Decimal range", ToDataType::family_name);
-            else
-                return ReturnType(false);
-        }
-
-        result = static_cast<ToNativeType>(out);
-        return ReturnType(true);
-    }
-    else
-    {
-        if constexpr (is_big_int_v<FromFieldType>)
-            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal256>, ToDataType, ReturnType>(static_cast<Int256>(value), 0, scale, result));
-        else if constexpr (std::is_same_v<FromFieldType, UInt64>)
-            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal128>, ToDataType, ReturnType>(static_cast<Int128>(value), 0, scale, result));
-        else
-            return ReturnType(convertDecimalsImpl<DataTypeDecimal<Decimal64>, ToDataType, ReturnType>(static_cast<Int64>(value), 0, scale, result));
-    }
-}
 
 template <typename FromDataType, typename ToDataType>
 requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
-inline typename ToDataType::FieldType convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
-{
-    typename ToDataType::FieldType result;
-    convertToDecimalImpl<FromDataType, ToDataType, void>(value, scale, result);
-    return result;
-}
+typename ToDataType::FieldType convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale);
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template typename TO_DATA_TYPE::FieldType convertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
 
 template <typename FromDataType, typename ToDataType>
 requires (is_arithmetic_v<typename FromDataType::FieldType> && IsDataTypeDecimal<ToDataType>)
-inline bool tryConvertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result)
-{
-    return convertToDecimalImpl<FromDataType, ToDataType, bool>(value, scale, result);
-}
+bool tryConvertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale, typename ToDataType::FieldType& result);
+
+#define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
+    extern template bool tryConvertToDecimal<FROM_DATA_TYPE, TO_DATA_TYPE>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale, typename TO_DATA_TYPE::FieldType& result);
+#define INVOKE(X) FOR_EACH_ARITHMETIC_TYPE_PASS(DISPATCH, X)
+FOR_EACH_DECIMAL_TYPE(INVOKE);
+#undef INVOKE
+#undef DISPATCH
+
 
 template <typename T>
-inline DataTypePtr createDecimalMaxPrecision(UInt64 scale)
-{
-    return std::make_shared<DataTypeDecimal<T>>(DecimalUtils::max_precision<T>, scale);
-}
+DataTypePtr createDecimalMaxPrecision(UInt64 scale);
+
+extern template DataTypePtr createDecimalMaxPrecision<Decimal32>(UInt64 scale);
+extern template DataTypePtr createDecimalMaxPrecision<Decimal64>(UInt64 scale);
+extern template DataTypePtr createDecimalMaxPrecision<Decimal128>(UInt64 scale);
+extern template DataTypePtr createDecimalMaxPrecision<Decimal256>(UInt64 scale);
+
+extern template class DataTypeDecimal<Decimal32>;
+extern template class DataTypeDecimal<Decimal64>;
+extern template class DataTypeDecimal<Decimal128>;
+extern template class DataTypeDecimal<Decimal256>;
 
 }

src/DataTypes/DataTypesNumber.cpp

@@ -102,4 +102,21 @@ void registerDataTypeNumbers(DataTypeFactory & factory)
     factory.registerAlias("DOUBLE PRECISION", "Float64", DataTypeFactory::CaseInsensitive);
 }
 
+/// Explicit template instantiations.
+template class DataTypeNumber<UInt8>;
+template class DataTypeNumber<UInt16>;
+template class DataTypeNumber<UInt32>;
+template class DataTypeNumber<UInt64>;
+template class DataTypeNumber<Int8>;
+template class DataTypeNumber<Int16>;
+template class DataTypeNumber<Int32>;
+template class DataTypeNumber<Int64>;
+template class DataTypeNumber<Float32>;
+template class DataTypeNumber<Float64>;
+
+template class DataTypeNumber<UInt128>;
+template class DataTypeNumber<Int128>;
+template class DataTypeNumber<UInt256>;
+template class DataTypeNumber<Int256>;
+
 }

src/DataTypes/DataTypesNumber.h

@@ -55,6 +55,22 @@ private:
     bool unsigned_can_be_signed = false;
 };
 
+extern template class DataTypeNumber<UInt8>;
+extern template class DataTypeNumber<UInt16>;
+extern template class DataTypeNumber<UInt32>;
+extern template class DataTypeNumber<UInt64>;
+extern template class DataTypeNumber<Int8>;
+extern template class DataTypeNumber<Int16>;
+extern template class DataTypeNumber<Int32>;
+extern template class DataTypeNumber<Int64>;
+extern template class DataTypeNumber<Float32>;
+extern template class DataTypeNumber<Float64>;
+
+extern template class DataTypeNumber<UInt128>;
+extern template class DataTypeNumber<Int128>;
+extern template class DataTypeNumber<UInt256>;
+extern template class DataTypeNumber<Int256>;
+
 using DataTypeUInt8 = DataTypeNumber<UInt8>;
 using DataTypeUInt16 = DataTypeNumber<UInt16>;
 using DataTypeUInt32 = DataTypeNumber<UInt32>;

src/DataTypes/IDataType.cpp

@@ -267,4 +267,91 @@ SerializationPtr IDataType::getSerialization(const NameAndTypePair & column)
     return column.type->getDefaultSerialization();
 }
 
+#define FOR_TYPES_OF_TYPE(M) \
+    M(TypeIndex) \
+    M(const IDataType &) \
+    M(const DataTypePtr &) \
+    M(WhichDataType)
+
+#define DISPATCH(TYPE) \
+bool isUInt8(TYPE data_type) { return WhichDataType(data_type).isUInt8(); } \
+bool isUInt16(TYPE data_type) { return WhichDataType(data_type).isUInt16(); } \
+bool isUInt32(TYPE data_type) { return WhichDataType(data_type).isUInt32(); } \
+bool isUInt64(TYPE data_type) { return WhichDataType(data_type).isUInt64(); } \
+bool isNativeUInt(TYPE data_type) { return WhichDataType(data_type).isNativeUInt(); } \
+bool isUInt(TYPE data_type) { return WhichDataType(data_type).isUInt(); } \
+\
+bool isInt8(TYPE data_type) { return WhichDataType(data_type).isInt8(); } \
+bool isInt16(TYPE data_type) { return WhichDataType(data_type).isInt16(); } \
+bool isInt32(TYPE data_type) { return WhichDataType(data_type).isInt32(); } \
+bool isInt64(TYPE data_type) { return WhichDataType(data_type).isInt64(); } \
+bool isNativeInt(TYPE data_type) { return WhichDataType(data_type).isNativeInt(); } \
+bool isInt(TYPE data_type) { return WhichDataType(data_type).isInt(); } \
+\
+bool isInteger(TYPE data_type) { return WhichDataType(data_type).isInteger(); } \
+bool isNativeInteger(TYPE data_type) { return WhichDataType(data_type).isNativeInteger(); } \
+\
+bool isDecimal(TYPE data_type) { return WhichDataType(data_type).isDecimal(); } \
+\
+bool isFloat(TYPE data_type) { return WhichDataType(data_type).isFloat(); } \
+\
+bool isNativeNumber(TYPE data_type) { return WhichDataType(data_type).isNativeNumber(); } \
+bool isNumber(TYPE data_type) { return WhichDataType(data_type).isNumber(); } \
+\
+bool isEnum8(TYPE data_type) { return WhichDataType(data_type).isEnum8(); } \
+bool isEnum16(TYPE data_type) { return WhichDataType(data_type).isEnum16(); } \
+bool isEnum(TYPE data_type) { return WhichDataType(data_type).isEnum(); } \
+\
+bool isDate(TYPE data_type) { return WhichDataType(data_type).isDate(); } \
+bool isDate32(TYPE data_type) { return WhichDataType(data_type).isDate32(); } \
+bool isDateOrDate32(TYPE data_type) { return WhichDataType(data_type).isDateOrDate32(); } \
+bool isDateTime(TYPE data_type) { return WhichDataType(data_type).isDateTime(); } \
+bool isDateTime64(TYPE data_type) { return WhichDataType(data_type).isDateTime64(); } \
+bool isDateTimeOrDateTime64(TYPE data_type) { return WhichDataType(data_type).isDateTimeOrDateTime64(); } \
+bool isDateOrDate32OrDateTimeOrDateTime64(TYPE data_type) { return WhichDataType(data_type).isDateOrDate32OrDateTimeOrDateTime64(); } \
+\
+bool isString(TYPE data_type) { return WhichDataType(data_type).isString(); } \
+bool isFixedString(TYPE data_type) { return WhichDataType(data_type).isFixedString(); } \
+bool isStringOrFixedString(TYPE data_type) { return WhichDataType(data_type).isStringOrFixedString(); } \
+\
+bool isUUID(TYPE data_type) { return WhichDataType(data_type).isUUID(); } \
+bool isIPv4(TYPE data_type) { return WhichDataType(data_type).isIPv4(); } \
+bool isIPv6(TYPE data_type) { return WhichDataType(data_type).isIPv6(); } \
+bool isArray(TYPE data_type) { return WhichDataType(data_type).isArray(); } \
+bool isTuple(TYPE data_type) { return WhichDataType(data_type).isTuple(); } \
+bool isMap(TYPE data_type) {return WhichDataType(data_type).isMap(); } \
+bool isInterval(TYPE data_type) {return WhichDataType(data_type).isInterval(); } \
+bool isObject(TYPE data_type) { return WhichDataType(data_type).isObject(); } \
+bool isVariant(TYPE data_type) { return WhichDataType(data_type).isVariant(); } \
+bool isNothing(TYPE data_type) { return WhichDataType(data_type).isNothing(); } \
+\
+bool isColumnedAsNumber(TYPE data_type) \
+{ \
+    WhichDataType which(data_type); \
+    return which.isInteger() || which.isFloat() || which.isDateOrDate32OrDateTimeOrDateTime64() || which.isUUID() || which.isIPv4() || which.isIPv6(); \
+} \
+\
+bool isColumnedAsDecimal(TYPE data_type) \
+{ \
+    WhichDataType which(data_type); \
+    return which.isDecimal() || which.isDateTime64(); \
+} \
+\
+bool isNotCreatable(TYPE data_type) \
+{ \
+    WhichDataType which(data_type); \
+    return which.isNothing() || which.isFunction() || which.isSet(); \
+} \
+\
+bool isNotDecimalButComparableToDecimal(TYPE data_type) \
+{ \
+    WhichDataType which(data_type); \
+    return which.isInt() || which.isUInt() || which.isFloat(); \
+} \
+
+FOR_TYPES_OF_TYPE(DISPATCH)
+
+#undef DISPATCH
+#undef FOR_TYPES_OF_TYPE
+
 }

src/DataTypes/IDataType.h

@@ -424,71 +424,76 @@ struct WhichDataType
 
 /// IDataType helpers (alternative for IDataType virtual methods with single point of truth)
 
-template <typename T> inline bool isUInt8(const T & data_type) { return WhichDataType(data_type).isUInt8(); }
-template <typename T> inline bool isUInt16(const T & data_type) { return WhichDataType(data_type).isUInt16(); }
-template <typename T> inline bool isUInt32(const T & data_type) { return WhichDataType(data_type).isUInt32(); }
-template <typename T> inline bool isUInt64(const T & data_type) { return WhichDataType(data_type).isUInt64(); }
-template <typename T> inline bool isNativeUInt(const T & data_type) { return WhichDataType(data_type).isNativeUInt(); }
-template <typename T> inline bool isUInt(const T & data_type) { return WhichDataType(data_type).isUInt(); }
+#define FOR_TYPES_OF_TYPE(M) \
+    M(TypeIndex) \
+    M(const IDataType &) \
+    M(const DataTypePtr &) \
+    M(WhichDataType)
 
-template <typename T> inline bool isInt8(const T & data_type) { return WhichDataType(data_type).isInt8(); }
-template <typename T> inline bool isInt16(const T & data_type) { return WhichDataType(data_type).isInt16(); }
-template <typename T> inline bool isInt32(const T & data_type) { return WhichDataType(data_type).isInt32(); }
-template <typename T> inline bool isInt64(const T & data_type) { return WhichDataType(data_type).isInt64(); }
-template <typename T> inline bool isNativeInt(const T & data_type) { return WhichDataType(data_type).isNativeInt(); }
-template <typename T> inline bool isInt(const T & data_type) { return WhichDataType(data_type).isInt(); }
+#define DISPATCH(TYPE) \
+bool isUInt8(TYPE data_type); \
+bool isUInt16(TYPE data_type); \
+bool isUInt32(TYPE data_type); \
+bool isUInt64(TYPE data_type); \
+bool isNativeUInt(TYPE data_type); \
+bool isUInt(TYPE data_type); \
+\
+bool isInt8(TYPE data_type); \
+bool isInt16(TYPE data_type); \
+bool isInt32(TYPE data_type); \
+bool isInt64(TYPE data_type); \
+bool isNativeInt(TYPE data_type); \
+bool isInt(TYPE data_type); \
+\
+bool isInteger(TYPE data_type); \
+bool isNativeInteger(TYPE data_type); \
+\
+bool isDecimal(TYPE data_type); \
+\
+bool isFloat(TYPE data_type); \
+\
+bool isNativeNumber(TYPE data_type); \
+bool isNumber(TYPE data_type); \
+\
+bool isEnum8(TYPE data_type); \
+bool isEnum16(TYPE data_type); \
+bool isEnum(TYPE data_type); \
+\
+bool isDate(TYPE data_type); \
+bool isDate32(TYPE data_type); \
+bool isDateOrDate32(TYPE data_type); \
+bool isDateTime(TYPE data_type); \
+bool isDateTime64(TYPE data_type); \
+bool isDateTimeOrDateTime64(TYPE data_type); \
+bool isDateOrDate32OrDateTimeOrDateTime64(TYPE data_type); \
+\
+bool isString(TYPE data_type); \
+bool isFixedString(TYPE data_type); \
+bool isStringOrFixedString(TYPE data_type); \
+\
+bool isUUID(TYPE data_type); \
+bool isIPv4(TYPE data_type); \
+bool isIPv6(TYPE data_type); \
+bool isArray(TYPE data_type); \
+bool isTuple(TYPE data_type); \
+bool isMap(TYPE data_type); \
+bool isInterval(TYPE data_type); \
+bool isObject(TYPE data_type); \
+bool isVariant(TYPE data_type); \
+bool isNothing(TYPE data_type); \
+\
+bool isColumnedAsNumber(TYPE data_type); \
+\
+bool isColumnedAsDecimal(TYPE data_type); \
+\
+bool isNotCreatable(TYPE data_type); \
+\
+bool isNotDecimalButComparableToDecimal(TYPE data_type); \
 
-template <typename T> inline bool isInteger(const T & data_type) { return WhichDataType(data_type).isInteger(); }
-template <typename T> inline bool isNativeInteger(const T & data_type) { return WhichDataType(data_type).isNativeInteger(); }
+FOR_TYPES_OF_TYPE(DISPATCH)
 
-template <typename T> inline bool isDecimal(const T & data_type) { return WhichDataType(data_type).isDecimal(); }
-
-template <typename T> inline bool isFloat(const T & data_type) { return WhichDataType(data_type).isFloat(); }
-
-template <typename T> inline bool isNativeNumber(const T & data_type) { return WhichDataType(data_type).isNativeNumber(); }
-template <typename T> inline bool isNumber(const T & data_type) { return WhichDataType(data_type).isNumber(); }
-
-template <typename T> inline bool isEnum8(const T & data_type) { return WhichDataType(data_type).isEnum8(); }
-template <typename T> inline bool isEnum16(const T & data_type) { return WhichDataType(data_type).isEnum16(); }
-template <typename T> inline bool isEnum(const T & data_type) { return WhichDataType(data_type).isEnum(); }
-
-template <typename T> inline bool isDate(const T & data_type) { return WhichDataType(data_type).isDate(); }
-template <typename T> inline bool isDate32(const T & data_type) { return WhichDataType(data_type).isDate32(); }
-template <typename T> inline bool isDateOrDate32(const T & data_type) { return WhichDataType(data_type).isDateOrDate32(); }
-template <typename T> inline bool isDateTime(const T & data_type) { return WhichDataType(data_type).isDateTime(); }
-template <typename T> inline bool isDateTime64(const T & data_type) { return WhichDataType(data_type).isDateTime64(); }
-template <typename T> inline bool isDateTimeOrDateTime64(const T & data_type) { return WhichDataType(data_type).isDateTimeOrDateTime64(); }
-template <typename T> inline bool isDateOrDate32OrDateTimeOrDateTime64(const T & data_type) { return WhichDataType(data_type).isDateOrDate32OrDateTimeOrDateTime64(); }
-
-template <typename T> inline bool isString(const T & data_type) { return WhichDataType(data_type).isString(); }
-template <typename T> inline bool isFixedString(const T & data_type) { return WhichDataType(data_type).isFixedString(); }
-template <typename T> inline bool isStringOrFixedString(const T & data_type) { return WhichDataType(data_type).isStringOrFixedString(); }
-
-template <typename T> inline bool isUUID(const T & data_type) { return WhichDataType(data_type).isUUID(); }
-template <typename T> inline bool isIPv4(const T & data_type) { return WhichDataType(data_type).isIPv4(); }
-template <typename T> inline bool isIPv6(const T & data_type) { return WhichDataType(data_type).isIPv6(); }
-template <typename T> inline bool isArray(const T & data_type) { return WhichDataType(data_type).isArray(); }
-template <typename T> inline bool isTuple(const T & data_type) { return WhichDataType(data_type).isTuple(); }
-template <typename T> inline bool isMap(const T & data_type) {return WhichDataType(data_type).isMap(); }
-template <typename T> inline bool isInterval(const T & data_type) {return WhichDataType(data_type).isInterval(); }
-template <typename T> inline bool isObject(const T & data_type) { return WhichDataType(data_type).isObject(); }
-template <typename T> inline bool isVariant(const T & data_type) { return WhichDataType(data_type).isVariant(); }
-
-template <typename T> inline bool isNothing(const T & data_type) { return WhichDataType(data_type).isNothing(); }
-
-template <typename T>
-inline bool isColumnedAsNumber(const T & data_type)
-{
-    WhichDataType which(data_type);
-    return which.isInteger() || which.isFloat() || which.isDateOrDate32OrDateTimeOrDateTime64() || which.isUUID() || which.isIPv4() || which.isIPv6();
-}
-
-template <typename T>
-inline bool isColumnedAsDecimal(const T & data_type)
-{
-    WhichDataType which(data_type);
-    return which.isDecimal() || which.isDateTime64();
-}
+#undef DISPATCH
+#undef FOR_TYPES_OF_TYPE
 
 // Same as isColumnedAsDecimal but also checks value type of underlyig column.
 template <typename T, typename DataType>
@@ -498,19 +503,6 @@ inline bool isColumnedAsDecimalT(const DataType & data_type)
     return (which.isDecimal() || which.isDateTime64()) && which.idx == TypeToTypeIndex<T>;
 }
 
-template <typename T>
-inline bool isNotCreatable(const T & data_type)
-{
-    WhichDataType which(data_type);
-    return which.isNothing() || which.isFunction() || which.isSet();
-}
-
-inline bool isNotDecimalButComparableToDecimal(const DataTypePtr & data_type)
-{
-    WhichDataType which(data_type);
-    return which.isInt() || which.isUInt() || which.isFloat();
-}
-
 inline bool isBool(const DataTypePtr & data_type)
 {
     return data_type->getName() == "Bool";

src/DataTypes/Serializations/SerializationDecimalBase.h

@@ -29,4 +29,10 @@ public:
     void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override;
 };
 
+extern template class SerializationDecimalBase<Decimal32>;
+extern template class SerializationDecimalBase<Decimal64>;
+extern template class SerializationDecimalBase<Decimal128>;
+extern template class SerializationDecimalBase<Decimal256>;
+extern template class SerializationDecimalBase<DateTime64>;
+
 }

src/Dictionaries/HashedArrayDictionary.cpp

@@ -1078,7 +1078,7 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::calculateBytesAllocate
                     bytes_allocated += container.allocated_bytes();
                 }
 
-                bucket_count = container.capacity();
+                bucket_count += container.capacity();
             }
         };
 
@@ -1089,6 +1089,13 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::calculateBytesAllocate
                 bytes_allocated += container.size();
     }
 
+    /// `bucket_count` should be a sum over all shards,
+    /// but it should not be a sum over all attributes, since it is used to
+    /// calculate load_factor like this: `element_count / bucket_count`
+    /// While element_count is a sum over all shards, not over all attributes.
+    if (attributes.size())
+        bucket_count /= attributes.size();
+
     if (update_field_loaded_block)
         bytes_allocated += update_field_loaded_block->allocatedBytes();
 
@@ -1167,17 +1174,24 @@ void registerDictionaryArrayHashed(DictionaryFactory & factory)
         if (shards <= 0 || 128 < shards)
             throw Exception(ErrorCodes::BAD_ARGUMENTS,"{}: SHARDS parameter should be within [1, 128]", full_name);
 
-        HashedArrayDictionaryStorageConfiguration configuration{require_nonempty, dict_lifetime, static_cast<size_t>(shards)};
+        Int64 shard_load_queue_backlog = config.getInt(config_prefix + dictionary_layout_prefix + ".shard_load_queue_backlog", 10000);
+        if (shard_load_queue_backlog <= 0)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: SHARD_LOAD_QUEUE_BACKLOG parameter should be greater then zero", full_name);
 
         if (source_ptr->hasUpdateField() && shards > 1)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: SHARDS parameter does not supports for updatable source (UPDATE_FIELD)", full_name);
 
+        HashedArrayDictionaryStorageConfiguration configuration{require_nonempty, dict_lifetime, static_cast<size_t>(shards), static_cast<UInt64>(shard_load_queue_backlog)};
+
         ContextMutablePtr context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix);
         const auto & settings = context->getSettingsRef();
 
         const auto * clickhouse_source = dynamic_cast<const ClickHouseDictionarySource *>(source_ptr.get());
         configuration.use_async_executor = clickhouse_source && clickhouse_source->isLocal() && settings.dictionary_use_async_executor;
 
+        if (settings.max_execution_time.totalSeconds() > 0)
+            configuration.load_timeout = std::chrono::seconds(settings.max_execution_time.totalSeconds());
+
         if (dictionary_key_type == DictionaryKeyType::Simple)
         {
             if (shards > 1)

src/Dictionaries/HashedArrayDictionary.h

@@ -29,6 +29,7 @@ struct HashedArrayDictionaryStorageConfiguration
     size_t shards = 1;
     size_t shard_load_queue_backlog = 10000;
     bool use_async_executor = false;
+    std::chrono::seconds load_timeout{0};
 };
 
 template <DictionaryKeyType dictionary_key_type, bool sharded>

src/Dictionaries/HashedDictionary.h

@@ -67,6 +67,7 @@ struct HashedDictionaryConfiguration
     const bool require_nonempty;
     const DictionaryLifetime lifetime;
     bool use_async_executor = false;
+    const std::chrono::seconds load_timeout{0};
 };
 
 template <DictionaryKeyType dictionary_key_type, bool sparse, bool sharded>

src/Dictionaries/HashedDictionaryParallelLoader.h

@@ -31,6 +31,7 @@ template <DictionaryKeyType dictionary_key_type, bool sparse, bool sharded> clas
 namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
+    extern const int TIMEOUT_EXCEEDED;
 }
 
 }
@@ -50,9 +51,10 @@ public:
         , shards(dictionary.configuration.shards)
         , pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, CurrentMetrics::HashedDictionaryThreadsScheduled, shards)
         , shards_queues(shards)
+        , loading_timeout(dictionary.configuration.load_timeout)
     {
         UInt64 backlog = dictionary.configuration.shard_load_queue_backlog;
-        LOG_TRACE(dictionary.log, "Will load the {} dictionary using {} threads (with {} backlog)", dictionary_name, shards, backlog);
+        LOG_TRACE(dictionary.log, "Will load the {} dictionary using {} threads (with {} backlog and timeout {} sec)", dictionary_name, shards, backlog, loading_timeout.count());
 
         shards_slots.resize(shards);
         iota(shards_slots.data(), shards_slots.size(), UInt64(0));
@@ -62,7 +64,11 @@ public:
             shards_queues[shard].emplace(backlog);
             pool.scheduleOrThrowOnError([this, shard, thread_group = CurrentThread::getGroup()]
             {
+                WorkerStatistic statistic;
                 SCOPE_EXIT_SAFE(
+                    LOG_TRACE(dictionary.log, "Finished worker for dictionary {} shard {}, processed {} blocks, {} rows, total time {}ms",
+                        dictionary_name, shard, statistic.total_blocks, statistic.total_rows, statistic.total_elapsed_ms);
+
                     if (thread_group)
                         CurrentThread::detachFromGroupIfNotDetached();
                 );
@@ -74,7 +80,9 @@ public:
                     CurrentThread::attachToGroupIfDetached(thread_group);
                 setThreadName("HashedDictLoad");
 
-                threadWorker(shard);
+                LOG_TRACE(dictionary.log, "Starting worker for dictionary {}, shard {}", dictionary_name, shard);
+
+                threadWorker(shard, statistic);
             });
         }
     }
@@ -87,8 +95,28 @@ public:
 
         for (size_t shard = 0; shard < shards; ++shard)
         {
-            if (!shards_queues[shard]->push(std::move(shards_blocks[shard])))
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not push to shards queue #{}", shard);
+            const auto & current_block = shards_blocks[shard];
+            while (!shards_queues[shard]->tryPush(current_block, /* milliseconds= */ 100))
+            {
+                if (shards_queues[shard]->isFinished())
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not push to finished shards queue #{}, dictionary {}", shard, dictionary_name);
+
+                /// We need to check if some workers failed
+                if (pool.active() != shards)
+                {
+                    LOG_DEBUG(dictionary.log, "Some workers for dictionary {} failed, stopping all workers", dictionary_name);
+                    stop_all_workers = true;
+                    pool.wait(); /// We expect exception to be thrown from the failed worker thread
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Worker threads for dictionary {} are not active", dictionary_name);
+                }
+
+                if (loading_timeout.count() && std::chrono::milliseconds(total_loading_time.elapsedMilliseconds()) > loading_timeout)
+                {
+                    stop_all_workers = true;
+                    pool.wait();
+                    throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout {} sec for dictionary {} loading is expired", loading_timeout.count(), dictionary_name);
+                }
+            }
         }
     }
 
@@ -124,27 +152,49 @@ private:
     String dictionary_name;
     const size_t shards;
     ThreadPool pool;
+    std::atomic_bool stop_all_workers{false};
     std::vector<std::optional<ConcurrentBoundedQueue<Block>>> shards_queues;
+    std::chrono::seconds loading_timeout;
+    Stopwatch total_loading_time;
+
     std::vector<UInt64> shards_slots;
     DictionaryKeysArenaHolder<dictionary_key_type> arena_holder;
 
-    void threadWorker(size_t shard)
+    struct WorkerStatistic
+    {
+        UInt64 total_elapsed_ms = 0;
+        UInt64 total_blocks = 0;
+        UInt64 total_rows = 0;
+    };
+
+    void threadWorker(size_t shard, WorkerStatistic & statistic)
     {
         Block block;
         DictionaryKeysArenaHolder<dictionary_key_type> arena_holder_;
         auto & shard_queue = *shards_queues[shard];
 
-        while (shard_queue.pop(block))
+        while (true)
         {
+            if (!shard_queue.tryPop(block, /* milliseconds= */ 100))
+            {
+                /// Check if we need to stop
+                if (stop_all_workers || shard_queue.isFinished())
+                    break;
+                /// Timeout expired, but the queue is not finished yet, try again
+                continue;
+            }
+
             Stopwatch watch;
             dictionary.blockToAttributes(block, arena_holder_, shard);
             UInt64 elapsed_ms = watch.elapsedMilliseconds();
-            if (elapsed_ms > 1'000)
-                LOG_TRACE(dictionary.log, "Block processing for shard #{} is slow {}ms (rows {}).", shard, elapsed_ms, block.rows());
-        }
 
-        if (!shard_queue.isFinished())
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not pull non finished shards queue #{}", shard);
+            statistic.total_elapsed_ms += elapsed_ms;
+            statistic.total_blocks += 1;
+            statistic.total_rows += block.rows();
+
+            if (elapsed_ms > 1'000)
+                LOG_TRACE(dictionary.log, "Block processing for shard #{} is slow {}ms (rows {})", shard, elapsed_ms, block.rows());
+        }
     }
 
     /// Split block to shards smaller block, using 'selector'.

src/Dictionaries/registerHashedDictionary.cpp

@@ -77,6 +77,7 @@ void registerDictionaryHashed(DictionaryFactory & factory)
             require_nonempty,
             dict_lifetime,
             use_async_executor,
+            std::chrono::seconds(settings.max_execution_time.totalSeconds()),
         };
 
         if (source_ptr->hasUpdateField() && shards > 1)

src/Functions/FunctionBase64Conversion.h

@@ -100,7 +100,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors mandatory_arguments{
-            {"value", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"}
+            {"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_arguments);

src/Functions/FunctionHelpers.h

@@ -108,8 +108,10 @@ struct FunctionArgumentDescriptor
 {
     const char * argument_name;
 
-    std::function<bool (const IDataType &)> type_validator_func;
-    std::function<bool (const IColumn &)> column_validator_func;
+    using TypeValidator = bool (*)(const IDataType &);
+    TypeValidator type_validator_func;
+    using ColumnValidator = bool (*)(const IColumn &);
+    ColumnValidator column_validator_func;
 
     const char * expected_type_description;
 

src/Functions/FunctionStringReplace.h

@@ -35,9 +35,9 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"haystack", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"},
-            {"pattern", &isString<IDataType>, nullptr, "String"},
-            {"replacement", &isString<IDataType>, nullptr, "String"}
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"},
+            {"pattern", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
+            {"replacement", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, args);

src/Functions/FunctionTokens.h

@@ -184,12 +184,12 @@ static inline void checkArgumentsWithSeparatorAndOptionalMaxSubstrings(
     const IFunction & func, const ColumnsWithTypeAndName & arguments)
 {
     FunctionArgumentDescriptors mandatory_args{
-        {"separator", &isString<IDataType>, isColumnConst, "const String"},
-        {"s", &isString<IDataType>, nullptr, "String"}
+        {"separator", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"},
+        {"s", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
     };
 
     FunctionArgumentDescriptors optional_args{
-        {"max_substrings", &isNativeInteger<IDataType>, isColumnConst, "const Number"},
+        {"max_substrings", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), isColumnConst, "const Number"},
     };
 
     validateFunctionArgumentTypes(func, arguments, mandatory_args, optional_args);
@@ -198,11 +198,11 @@ static inline void checkArgumentsWithSeparatorAndOptionalMaxSubstrings(
 static inline void checkArgumentsWithOptionalMaxSubstrings(const IFunction & func, const ColumnsWithTypeAndName & arguments)
 {
     FunctionArgumentDescriptors mandatory_args{
-        {"s", &isString<IDataType>, nullptr, "String"},
+        {"s", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
     };
 
     FunctionArgumentDescriptors optional_args{
-        {"max_substrings", &isNativeInteger<IDataType>, isColumnConst, "const Number"},
+        {"max_substrings", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), isColumnConst, "const Number"},
     };
 
     validateFunctionArgumentTypes(func, arguments, mandatory_args, optional_args);

src/Functions/FunctionUnixTimestamp64.h

@@ -45,7 +45,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"value", &isDateTime64<IDataType>, nullptr, "DateTime64"}
+            {"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime64), nullptr, "DateTime64"}
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/FunctionsAES.h

@@ -154,21 +154,21 @@ private:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         auto optional_args = FunctionArgumentDescriptors{
-            {"IV", &isStringOrFixedString<IDataType>, nullptr, "Initialization vector binary string"},
+            {"IV", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Initialization vector binary string"},
         };
 
         if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL)
         {
             optional_args.emplace_back(FunctionArgumentDescriptor{
-                "AAD", &isStringOrFixedString<IDataType>, nullptr, "Additional authenticated data binary string for GCM mode"
+                "AAD", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Additional authenticated data binary string for GCM mode"
             });
         }
 
         validateFunctionArgumentTypes(*this, arguments,
             FunctionArgumentDescriptors{
-                {"mode", &isStringOrFixedString<IDataType>, isColumnConst, "encryption mode string"},
-                {"input", &isStringOrFixedString<IDataType>, {}, "plaintext"},
-                {"key", &isStringOrFixedString<IDataType>, {}, "encryption key binary string"},
+                {"mode", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "encryption mode string"},
+                {"input", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "plaintext"},
+                {"key", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "encryption key binary string"},
             },
             optional_args
         );
@@ -425,21 +425,21 @@ private:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         auto optional_args = FunctionArgumentDescriptors{
-            {"IV", &isStringOrFixedString<IDataType>, nullptr, "Initialization vector binary string"},
+            {"IV", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Initialization vector binary string"},
         };
 
         if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL)
         {
             optional_args.emplace_back(FunctionArgumentDescriptor{
-                "AAD", &isStringOrFixedString<IDataType>, nullptr, "Additional authenticated data binary string for GCM mode"
+                "AAD", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "Additional authenticated data binary string for GCM mode"
             });
         }
 
         validateFunctionArgumentTypes(*this, arguments,
             FunctionArgumentDescriptors{
-                {"mode", &isStringOrFixedString<IDataType>, isColumnConst, "decryption mode string"},
-                {"input", &isStringOrFixedString<IDataType>, {}, "ciphertext"},
-                {"key", &isStringOrFixedString<IDataType>, {}, "decryption key binary string"},
+                {"mode", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "decryption mode string"},
+                {"input", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "ciphertext"},
+                {"key", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), {}, "decryption key binary string"},
             },
             optional_args
         );

src/Functions/FunctionsConversion.h

@@ -2129,12 +2129,12 @@ public:
 
         if constexpr (to_decimal)
         {
-            mandatory_args.push_back({"scale", &isNativeInteger<IDataType>, &isColumnConst, "const Integer"});
+            mandatory_args.push_back({"scale", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), &isColumnConst, "const Integer"});
         }
 
         if (!to_decimal && isDateTime64<Name, ToDataType>(arguments))
         {
-            mandatory_args.push_back({"scale", &isNativeInteger<IDataType>, &isColumnConst, "const Integer"});
+            mandatory_args.push_back({"scale", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), &isColumnConst, "const Integer"});
         }
 
         // toString(DateTime or DateTime64, [timezone: String])
@@ -2150,7 +2150,7 @@ public:
             // toDateTime64(value, scale : Integer[, timezone: String])
             || std::is_same_v<ToDataType, DataTypeDateTime64>)
         {
-            optional_args.push_back({"timezone", &isString<IDataType>, nullptr, "String"});
+            optional_args.push_back({"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"});
         }
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
@@ -2498,11 +2498,11 @@ public:
         if (isDateTime64<Name, ToDataType>(arguments))
         {
             validateFunctionArgumentTypes(*this, arguments,
-                FunctionArgumentDescriptors{{"string", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"}},
+                FunctionArgumentDescriptors{{"string", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"}},
                 // optional
                 FunctionArgumentDescriptors{
-                    {"precision", &isUInt8<IDataType>, isColumnConst, "const UInt8"},
-                    {"timezone", &isStringOrFixedString<IDataType>, isColumnConst, "const String or FixedString"},
+                    {"precision", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt8), isColumnConst, "const UInt8"},
+                    {"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "const String or FixedString"},
                 });
 
             UInt64 scale = to_datetime64 ? DataTypeDateTime64::default_scale : 0;

src/Functions/JSONArrayLength.cpp

@@ -45,7 +45,7 @@ namespace
         DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
         {
             auto args = FunctionArgumentDescriptors{
-                {"json", &isString<IDataType>, nullptr, "String"},
+                {"json", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
             };
 
             validateFunctionArgumentTypes(*this, arguments, args);

src/Functions/URL/URLHierarchy.cpp

@@ -27,7 +27,7 @@ public:
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"URL", &isString<IDataType>, nullptr, "String"},
+            {"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
         };
 
         validateFunctionArgumentTypes(func, arguments, mandatory_args);

src/Functions/URL/URLPathHierarchy.cpp

@@ -25,7 +25,7 @@ public:
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"URL", &isString<IDataType>, nullptr, "String"},
+            {"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
         };
 
         validateFunctionArgumentTypes(func, arguments, mandatory_args);

src/Functions/URL/extractURLParameterNames.cpp

@@ -25,7 +25,7 @@ public:
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"URL", &isString<IDataType>, nullptr, "String"},
+            {"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
         };
 
         validateFunctionArgumentTypes(func, arguments, mandatory_args);

src/Functions/URL/extractURLParameters.cpp

@@ -26,7 +26,7 @@ public:
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"URL", &isString<IDataType>, nullptr, "String"},
+            {"URL", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
         };
 
         validateFunctionArgumentTypes(func, arguments, mandatory_args);

src/Functions/array/arrayJaccardIndex.cpp

@@ -84,8 +84,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"array_1", &isArray<IDataType>, nullptr, "Array"},
-            {"array_2", &isArray<IDataType>, nullptr, "Array"},
+            {"array_1", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
+            {"array_2", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
         };
         validateFunctionArgumentTypes(*this, arguments, args);
         return std::make_shared<DataTypeNumber<ResultType>>();

src/Functions/array/arrayRandomSample.cpp

@@ -36,8 +36,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"array", &isArray<IDataType>, nullptr, "Array"},
-            {"samples", &isUInt<IDataType>, isColumnConst, "const UInt*"},
+            {"array", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
+            {"samples", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt), isColumnConst, "const UInt*"},
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/array/arrayShingles.cpp

@@ -28,8 +28,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
             FunctionArgumentDescriptors args{
-                {"array", &isArray<IDataType>, nullptr, "Array"},
-                {"length", &isInteger<IDataType>, nullptr, "Integer"}
+                {"array", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
+                {"length", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isInteger), nullptr, "Integer"}
             };
             validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/arrayStringConcat.cpp

@@ -151,12 +151,12 @@ public:
     {
         FunctionArgumentDescriptors mandatory_args
         {
-            {"arr", &isArray<IDataType>, nullptr, "Array"},
+            {"arr", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
         };
 
         FunctionArgumentDescriptors optional_args
         {
-            {"separator", &isString<IDataType>, isColumnConst, "const String"},
+            {"separator", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"},
         };
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);

src/Functions/castOrDefault.cpp

@@ -210,10 +210,10 @@ private:
         FunctionArgumentDescriptors optional_args;
 
         if constexpr (IsDataTypeDecimal<Type>)
-            mandatory_args.push_back({"scale", &isNativeInteger<IDataType>, &isColumnConst, "const Integer"});
+            mandatory_args.push_back({"scale", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), &isColumnConst, "const Integer"});
 
         if (std::is_same_v<Type, DataTypeDateTime> || std::is_same_v<Type, DataTypeDateTime64>)
-            optional_args.push_back({"timezone", &isString<IDataType>, isColumnConst, "const String"});
+            optional_args.push_back({"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"});
 
         optional_args.push_back({"default_value", nullptr, nullptr, nullptr});
 

src/Functions/countMatches.h

@@ -35,8 +35,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"haystack", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"},
-            {"pattern", &isString<IDataType>, isColumnConst, "constant String"}
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"},
+            {"pattern", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "constant String"}
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/extractAll.cpp

@@ -53,8 +53,8 @@ public:
     static void checkArguments(const IFunction & func, const ColumnsWithTypeAndName & arguments)
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"haystack", &isString<IDataType>, nullptr, "String"},
-            {"pattern", &isString<IDataType>, isColumnConst, "const String"}
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
+            {"pattern", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"}
         };
 
         validateFunctionArgumentTypes(func, arguments, mandatory_args);

src/Functions/extractAllGroups.h

@@ -71,8 +71,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"haystack", &isStringOrFixedString<IDataType>, nullptr, "const String or const FixedString"},
-            {"needle", &isStringOrFixedString<IDataType>, isColumnConst, "const String or const FixedString"},
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "const String or const FixedString"},
+            {"needle", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "const String or const FixedString"},
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/extractGroups.cpp

@@ -45,8 +45,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"haystack", &isStringOrFixedString<IDataType>, nullptr, "const String or const FixedString"},
-            {"needle", &isStringOrFixedString<IDataType>, isColumnConst, "const String or const FixedString"},
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "const String or const FixedString"},
+            {"needle", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), isColumnConst, "const String or const FixedString"},
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/formatQuery.cpp

@@ -54,7 +54,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"query", &isString<IDataType>, nullptr, "String"}
+            {"query", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/fromDaysSinceYearZero.cpp

@@ -52,7 +52,7 @@ public:
 
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
-        FunctionArgumentDescriptors args{{"days", &isNativeInteger<IDataType>, nullptr, "Integer"}};
+        FunctionArgumentDescriptors args{{"days", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), nullptr, "Integer"}};
 
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/makeDate.cpp

@@ -82,17 +82,17 @@ public:
         if (is_year_month_variant)
         {
             FunctionArgumentDescriptors args{
-                {mandatory_argument_names_year_month_day[0], &isNumber<IDataType>, nullptr, "Number"},
-                {mandatory_argument_names_year_month_day[1], &isNumber<IDataType>, nullptr, "Number"},
-                {mandatory_argument_names_year_month_day[2], &isNumber<IDataType>, nullptr, "Number"}
+                {mandatory_argument_names_year_month_day[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+                {mandatory_argument_names_year_month_day[1], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+                {mandatory_argument_names_year_month_day[2], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"}
             };
             validateFunctionArgumentTypes(*this, arguments, args);
         }
         else
         {
             FunctionArgumentDescriptors args{
-                {mandatory_argument_names_year_dayofyear[0], &isNumber<IDataType>, nullptr, "Number"},
-                {mandatory_argument_names_year_dayofyear[1], &isNumber<IDataType>, nullptr, "Number"}
+                {mandatory_argument_names_year_dayofyear[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+                {mandatory_argument_names_year_dayofyear[1], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"}
             };
             validateFunctionArgumentTypes(*this, arguments, args);
         }
@@ -189,7 +189,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {mandatory_argument_names[0], &isNumber<IDataType>, nullptr, "Number"}
+            {mandatory_argument_names[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, args);
@@ -344,16 +344,16 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors mandatory_args{
-            {mandatory_argument_names[0], &isNumber<IDataType>, nullptr, "Number"},
-            {mandatory_argument_names[1], &isNumber<IDataType>, nullptr, "Number"},
-            {mandatory_argument_names[2], &isNumber<IDataType>, nullptr, "Number"},
-            {mandatory_argument_names[3], &isNumber<IDataType>, nullptr, "Number"},
-            {mandatory_argument_names[4], &isNumber<IDataType>, nullptr, "Number"},
-            {mandatory_argument_names[5], &isNumber<IDataType>, nullptr, "Number"}
+            {mandatory_argument_names[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {mandatory_argument_names[1], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {mandatory_argument_names[2], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {mandatory_argument_names[3], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {mandatory_argument_names[4], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {mandatory_argument_names[5], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"}
         };
 
         FunctionArgumentDescriptors optional_args{
-            {optional_argument_names[0], &isString<IDataType>, isColumnConst, "const String"}
+            {optional_argument_names[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
@@ -425,18 +425,18 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors mandatory_args{
-            {mandatory_argument_names[0], &isNumber<IDataType>, nullptr, "Number"},
-            {mandatory_argument_names[1], &isNumber<IDataType>, nullptr, "Number"},
-            {mandatory_argument_names[2], &isNumber<IDataType>, nullptr, "Number"},
-            {mandatory_argument_names[3], &isNumber<IDataType>, nullptr, "Number"},
-            {mandatory_argument_names[4], &isNumber<IDataType>, nullptr, "Number"},
-            {mandatory_argument_names[5], &isNumber<IDataType>, nullptr, "Number"}
+            {mandatory_argument_names[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {mandatory_argument_names[1], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {mandatory_argument_names[2], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {mandatory_argument_names[3], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {mandatory_argument_names[4], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {mandatory_argument_names[5], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"}
         };
 
         FunctionArgumentDescriptors optional_args{
-            {optional_argument_names[0], &isNumber<IDataType>, nullptr, "const Number"},
-            {optional_argument_names[1], &isNumber<IDataType>, isColumnConst, "const Number"},
-            {optional_argument_names[2], &isString<IDataType>, isColumnConst, "const String"}
+            {optional_argument_names[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "const Number"},
+            {optional_argument_names[1], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), isColumnConst, "const Number"},
+            {optional_argument_names[2], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
@@ -564,11 +564,11 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors mandatory_args{
-            {mandatory_argument_names[0], &isNumber<IDataType>, nullptr, "Number"}
+            {mandatory_argument_names[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"}
         };
 
         FunctionArgumentDescriptors optional_args{
-            {optional_argument_names[0], &isString<IDataType>, isColumnConst, "const String"}
+            {optional_argument_names[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
@@ -643,12 +643,12 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors mandatory_args{
-            {mandatory_argument_names[0], &isNumber<IDataType>, nullptr, "Number"}
+            {mandatory_argument_names[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"}
         };
 
         FunctionArgumentDescriptors optional_args{
-            {optional_argument_names[0], &isNumber<IDataType>, isColumnConst, "const Number"},
-            {optional_argument_names[0], &isString<IDataType>, isColumnConst, "const String"}
+            {optional_argument_names[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), isColumnConst, "const Number"},
+            {optional_argument_names[0], static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);

src/Functions/multiIf.cpp

@@ -3,12 +3,20 @@
 #include <Columns/ColumnNullable.h>
 #include <Columns/ColumnConst.h>
 #include <Columns/ColumnsNumber.h>
+#include <Columns/ColumnDecimal.h>
 #include <Columns/MaskOperations.h>
 #include <Interpreters/castColumn.h>
 #include <Common/assert_cast.h>
 #include <Common/typeid_cast.h>
 #include <Interpreters/Context.h>
 #include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeEnum.h>
+#include <DataTypes/DataTypesDecimal.h>
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
+#include <DataTypes/DataTypeDateTime.h>
+#include <DataTypes/DataTypeDateTime64.h>
 #include <DataTypes/DataTypeVariant.h>
 #include <DataTypes/getLeastSupertype.h>
 
@@ -20,7 +28,7 @@ namespace ErrorCodes
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
     extern const int NOT_IMPLEMENTED;
-    extern const int LOGICAL_ERROR;
+    extern const int BAD_ARGUMENTS;
 }
 
 namespace
@@ -149,6 +157,10 @@ public:
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
+        /// Fast path when data is empty
+        if (input_rows_count == 0)
+            return result_type->createColumn();
+
         ColumnsWithTypeAndName arguments = args;
         executeShortCircuitArguments(arguments);
         /** We will gather values from columns in branches to result column,
@@ -249,64 +261,73 @@ public:
         }
 
         const WhichDataType which(removeNullable(result_type));
-        bool execute_multiif_columnar
-            = allow_execute_multiif_columnar && !contains_short && (which.isInt() || which.isUInt() || which.isFloat());
+        bool execute_multiif_columnar = allow_execute_multiif_columnar && !contains_short
+            && instructions.size() <= std::numeric_limits<UInt8>::max()
+            && (which.isInt() || which.isUInt() || which.isFloat() || which.isDecimal() || which.isDateOrDate32OrDateTimeOrDateTime64()
+                || which.isEnum() || which.isIPv4() || which.isIPv6());
 
         size_t rows = input_rows_count;
         if (!execute_multiif_columnar)
         {
             MutableColumnPtr res = return_type->createColumn();
+            res->reserve(rows);
             executeInstructions(instructions, rows, res);
             return std::move(res);
         }
 
-#define EXECUTE_INSTRUCTIONS_COLUMNAR(TYPE, INDEX) \
+#define EXECUTE_INSTRUCTIONS_COLUMNAR(TYPE, FIELD, INDEX) \
     if (which.is##TYPE()) \
     { \
-        MutableColumnPtr res = ColumnVector<TYPE>::create(rows); \
-        MutableColumnPtr null_map = result_type->isNullable() ? ColumnUInt8::create(rows) : nullptr; \
-        executeInstructionsColumnar<TYPE, INDEX>(instructions, rows, res, null_map, result_type->isNullable()); \
-        if (!result_type->isNullable()) \
-            return std::move(res); \
+        MutableColumnPtr res = result_type->createColumn(); \
+        if (result_type->isNullable()) \
+        { \
+            auto & res_nullable = assert_cast<ColumnNullable &>(*res); \
+            auto & res_data = assert_cast<ColumnVectorOrDecimal<FIELD> &>(res_nullable.getNestedColumn()).getData(); \
+            auto & res_null_map = res_nullable.getNullMapData(); \
+            executeInstructionsColumnar<FIELD, INDEX, true>(instructions, rows, res_data, &res_null_map); \
+        } \
         else \
-            return ColumnNullable::create(std::move(res), std::move(null_map)); \
+        { \
+            auto & res_data = assert_cast<ColumnVectorOrDecimal<FIELD> &>(*res).getData(); \
+            executeInstructionsColumnar<FIELD, INDEX, false>(instructions, rows, res_data, nullptr); \
+        } \
+        return std::move(res); \
     }
 
 #define ENUMERATE_NUMERIC_TYPES(M, INDEX) \
-    M(UInt8, INDEX) \
-    M(UInt16, INDEX) \
-    M(UInt32, INDEX) \
-    M(UInt64, INDEX) \
-    M(Int8, INDEX) \
-    M(Int16, INDEX) \
-    M(Int32, INDEX) \
-    M(Int64, INDEX) \
-    M(UInt128, INDEX) \
-    M(UInt256, INDEX) \
-    M(Int128, INDEX) \
-    M(Int256, INDEX) \
-    M(Float32, INDEX) \
-    M(Float64, INDEX) \
+    M(UInt8, UInt8, INDEX) \
+    M(UInt16, UInt16, INDEX) \
+    M(UInt32, UInt32, INDEX) \
+    M(UInt64, UInt64, INDEX) \
+    M(Int8, Int8, INDEX) \
+    M(Int16, Int16, INDEX) \
+    M(Int32, Int32, INDEX) \
+    M(Int64, Int64, INDEX) \
+    M(Float32, Float32, INDEX) \
+    M(Float64, Float64, INDEX) \
+    M(UInt128, UInt128, INDEX) \
+    M(UInt256, UInt256, INDEX) \
+    M(Int128, Int128, INDEX) \
+    M(Int256, Int256, INDEX) \
+    M(Decimal32, Decimal32, INDEX) \
+    M(Decimal64, Decimal64, INDEX) \
+    M(Decimal128, Decimal128, INDEX) \
+    M(Decimal256, Decimal256, INDEX) \
+    M(Date, UInt16, INDEX) \
+    M(Date32, Int32, INDEX) \
+    M(DateTime, UInt32, INDEX) \
+    M(DateTime64, DateTime64, INDEX) \
+    M(Enum8, Int8, INDEX) \
+    M(Enum16, Int16, INDEX) \
+    M(IPv4, IPv4, INDEX) \
+    M(IPv6, IPv6, INDEX) \
     throw Exception( \
         ErrorCodes::NOT_IMPLEMENTED, "Columnar execution of function {} not implemented for type {}", getName(), result_type->getName());
 
-        size_t num_instructions = instructions.size();
-        if (num_instructions <= std::numeric_limits<Int16>::max())
-        {
-            ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR, Int16)
-        }
-        else if (num_instructions <= std::numeric_limits<Int32>::max())
-        {
-            ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR, Int32)
-        }
-        else if (num_instructions <= std::numeric_limits<Int64>::max())
-        {
-            ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR, Int64)
-        }
-        else
-            throw Exception(
-                ErrorCodes::LOGICAL_ERROR, "Instruction size({}) of function {} is out of range", getName(), result_type->getName());
+        ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR, UInt8)
     }
+#undef ENUMERATE_NUMERIC_TYPES
+#undef EXECUTE_INSTRUCTIONS_COLUMNAR
 
 private:
 
@@ -348,11 +369,11 @@ private:
 
     /// We should read source from which instruction on each row?
     template <typename S>
-    static void calculateInserts(std::vector<Instruction> & instructions, size_t rows, PaddedPODArray<S> & inserts)
+    static NO_INLINE void calculateInserts(const std::vector<Instruction> & instructions, size_t rows, PaddedPODArray<S> & inserts)
     {
-        for (S i = static_cast<S>(instructions.size() - 1); i >= 0; --i)
+        for (S i = instructions.size() - 1; i != static_cast<S>(-1); --i)
         {
-            auto & instruction = instructions[i];
+            const auto & instruction = instructions[i];
             if (instruction.condition_always_true)
             {
                 for (size_t row_i = 0; row_i < rows; ++row_i)
@@ -388,60 +409,62 @@ private:
         }
     }
 
-    template <typename T, typename S>
-    static void executeInstructionsColumnar(std::vector<Instruction> & instructions, size_t rows, const MutableColumnPtr & res, const MutableColumnPtr & null_map, bool nullable)
+    template <typename T, typename S, bool nullable_result = false>
+    static NO_INLINE void executeInstructionsColumnar(
+        const std::vector<Instruction> & instructions,
+        size_t rows,
+        PaddedPODArray<T> & res_data,
+        PaddedPODArray<UInt8> * res_null_map = nullptr)
     {
         PaddedPODArray<S> inserts(rows, static_cast<S>(instructions.size()));
         calculateInserts(instructions, rows, inserts);
 
-        PaddedPODArray<T> & res_data = assert_cast<ColumnVector<T> &>(*res).getData();
-        if (!nullable)
+        res_data.resize_exact(rows);
+        if constexpr (nullable_result)
         {
-            for (size_t row_i = 0; row_i < rows; ++row_i)
+            if (!res_null_map)
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid result null_map while result type is nullable");
+
+            res_null_map->resize_exact(rows);
+        }
+
+        std::vector<const T *> data_cols(instructions.size(), nullptr);
+        std::vector<const UInt8 *> null_map_cols(instructions.size(), nullptr);
+        for (size_t i = 0; i < instructions.size(); ++i)
+        {
+            const auto & instruction = instructions[i];
+            const IColumn * non_const_col = instructions[i].source_is_constant
+                ? &assert_cast<const ColumnConst &>(*instruction.source).getDataColumn()
+                : instruction.source.get();
+            const ColumnNullable * nullable_col = checkAndGetColumn<ColumnNullable>(non_const_col);
+            data_cols[i] = nullable_col ? assert_cast<const ColumnVectorOrDecimal<T> &>(nullable_col->getNestedColumn()).getData().data()
+                                        : assert_cast<const ColumnVectorOrDecimal<T> &>(*non_const_col).getData().data();
+            null_map_cols[i] = nullable_col ? assert_cast<const ColumnUInt8 &>(nullable_col->getNullMapColumn()).getData().data() : nullptr;
+        }
+
+        std::unique_ptr<PaddedPODArray<UInt8>> shared_null_map;
+        if constexpr (nullable_result)
+        {
+            for (auto & col : null_map_cols)
             {
-                auto & instruction = instructions[inserts[row_i]];
-                auto ref = instruction.source->getDataAt(row_i);
-                res_data[row_i] = *reinterpret_cast<const T*>(ref.data);
+                if (!col)
+                {
+                    if (!shared_null_map)
+                        shared_null_map = std::make_unique<PaddedPODArray<UInt8>>(rows, 0);
+
+                    col = shared_null_map->data();
+                }
             }
         }
-        else
+
+        for (size_t row_i = 0; row_i < rows; ++row_i)
         {
-            PaddedPODArray<UInt8> & null_map_data = assert_cast<ColumnUInt8 &>(*null_map).getData();
-            std::vector<const T*> data_cols(instructions.size());
-            std::vector<const UInt8 *> null_map_cols(instructions.size());
-            ColumnPtr shared_null_map_col = nullptr;
-            for (size_t i = 0; i < instructions.size(); ++i)
-            {
-                if (instructions[i].source->isNullable())
-                {
-                    const ColumnNullable * nullable_col;
-                    if (!instructions[i].source_is_constant)
-                        nullable_col = assert_cast<const ColumnNullable *>(instructions[i].source.get());
-                    else
-                    {
-                        const ColumnPtr data_column = assert_cast<const ColumnConst &>(*instructions[i].source).getDataColumnPtr();
-                        nullable_col = assert_cast<const ColumnNullable *>(data_column.get());
-                    }
-                    null_map_cols[i] = assert_cast<const ColumnUInt8 &>(*nullable_col->getNullMapColumnPtr()).getData().data();
-                    data_cols[i] = assert_cast<const ColumnVector<T> &>(*nullable_col->getNestedColumnPtr()).getData().data();
-                }
-                else
-                {
-                    if (!shared_null_map_col)
-                    {
-                        shared_null_map_col = ColumnUInt8::create(rows, 0);
-                    }
-                    null_map_cols[i] = assert_cast<const ColumnUInt8 &>(*shared_null_map_col).getData().data();
-                    data_cols[i] = assert_cast<const ColumnVector<T> &>(*instructions[i].source).getData().data();
-                }
-            }
-            for (size_t row_i = 0; row_i < rows; ++row_i)
-            {
-                auto & instruction = instructions[inserts[row_i]];
-                size_t index = instruction.source_is_constant ? 0 : row_i;
-                res_data[row_i] = *(data_cols[inserts[row_i]] + index);
-                null_map_data[row_i] = *(null_map_cols[inserts[row_i]] + index);
-            }
+            S insert = inserts[row_i];
+            const auto & instruction = instructions[insert];
+            size_t index = instruction.source_is_constant ? 0 : row_i;
+            res_data[row_i] = *(data_cols[insert] + index);
+            if constexpr (nullable_result)
+                (*res_null_map)[row_i] = *(null_map_cols[insert] + index);
         }
     }
 

src/Functions/parseDateTime.cpp

@@ -489,12 +489,12 @@ namespace
         DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
         {
             FunctionArgumentDescriptors mandatory_args{
-                {"time", &isString<IDataType>, nullptr, "String"},
-                {"format", &isString<IDataType>, nullptr, "String"}
+                {"time", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
+                {"format", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
             };
 
             FunctionArgumentDescriptors optional_args{
-                {"timezone", &isString<IDataType>, &isColumnConst, "const String"}
+                {"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), &isColumnConst, "const String"}
             };
 
             validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);

src/Functions/regexpExtract.cpp

@@ -47,12 +47,12 @@ public:
                 arguments.size());
 
         FunctionArgumentDescriptors args{
-            {"haystack", &isString<IDataType>, nullptr, "String"},
-            {"pattern", &isString<IDataType>, isColumnConst, "const String"},
+            {"haystack", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
+            {"pattern", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), isColumnConst, "const String"},
         };
 
         if (arguments.size() == 3)
-            args.emplace_back(FunctionArgumentDescriptor{"index", &isInteger<IDataType>, nullptr, "Integer"});
+            args.emplace_back(FunctionArgumentDescriptor{"index", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isInteger), nullptr, "Integer"});
 
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/repeat.cpp

@@ -186,8 +186,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"s", &isString<IDataType>, nullptr, "String"},
-            {"n", &isInteger<IDataType>, nullptr, "Integer"},
+            {"s", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
+            {"n", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isInteger), nullptr, "Integer"},
         };
 
         validateFunctionArgumentTypes(*this, arguments, args);

src/Functions/seriesDecomposeSTL.cpp

@@ -42,8 +42,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"time_series", &isArray<IDataType>, nullptr, "Array"},
-            {"period", &isNativeUInt<IDataType>, nullptr, "Unsigned Integer"},
+            {"time_series", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"},
+            {"period", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), nullptr, "Unsigned Integer"},
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/seriesOutliersDetectTukey.cpp

@@ -45,11 +45,11 @@ public:
                 getName(),
                 arguments.size());
 
-        FunctionArgumentDescriptors mandatory_args{{"time_series", &isArray<IDataType>, nullptr, "Array"}};
+        FunctionArgumentDescriptors mandatory_args{{"time_series", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"}};
         FunctionArgumentDescriptors optional_args{
-            {"min_percentile", &isFloat<IDataType>, isColumnConst, "Number"},
-            {"max_percentile", &isFloat<IDataType>, isColumnConst, "Number"},
-            {"k", &isNativeNumber<IDataType>, isColumnConst, "Number"}};
+            {"min_percentile", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isFloat), isColumnConst, "Number"},
+            {"max_percentile", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isFloat), isColumnConst, "Number"},
+            {"k", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeNumber), isColumnConst, "Number"}};
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
 

src/Functions/seriesPeriodDetectFFT.cpp

@@ -52,7 +52,7 @@ public:
 
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
-        FunctionArgumentDescriptors args{{"time_series", &isArray<IDataType>, nullptr, "Array"}};
+        FunctionArgumentDescriptors args{{"time_series", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isArray), nullptr, "Array"}};
         validateFunctionArgumentTypes(*this, arguments, args);
 
         return std::make_shared<DataTypeFloat64>();

src/Functions/snowflake.cpp

@@ -47,7 +47,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"value", &isDateTime<IDataType>, nullptr, "DateTime"}
+            {"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime), nullptr, "DateTime"}
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 
@@ -91,10 +91,10 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"value", &isInt64<IDataType>, nullptr, "Int64"}
+            {"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isInt64), nullptr, "Int64"}
         };
         FunctionArgumentDescriptors optional_args{
-            {"time_zone", &isString<IDataType>, nullptr, "String"}
+            {"time_zone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
         };
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
 
@@ -151,7 +151,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"value", &isDateTime64<IDataType>, nullptr, "DateTime64"}
+            {"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime64), nullptr, "DateTime64"}
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 
@@ -203,10 +203,10 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"value", &isInt64<IDataType>, nullptr, "Int64"}
+            {"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isInt64), nullptr, "Int64"}
         };
         FunctionArgumentDescriptors optional_args{
-            {"time_zone", &isString<IDataType>, nullptr, "String"}
+            {"time_zone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
         };
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
 

src/Functions/space.cpp

@@ -45,7 +45,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"n", &isInteger<IDataType>, nullptr, "Integer"}
+            {"n", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isInteger), nullptr, "Integer"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, args);

src/Functions/sqid.cpp

@@ -98,7 +98,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors args{
-            {"sqid", &isString<IDataType>, nullptr, "String"}
+            {"sqid", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
         };
         validateFunctionArgumentTypes(*this, arguments, args);
 

src/Functions/timestamp.cpp

@@ -41,10 +41,10 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors mandatory_args{
-            {"timestamp", &isStringOrFixedString<IDataType>, nullptr, "String or FixedString"}
+            {"timestamp", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String or FixedString"}
         };
         FunctionArgumentDescriptors optional_args{
-            {"time", &isString<IDataType>, nullptr, "String"}
+            {"time", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
         };
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
 

src/Functions/toDecimalString.cpp

@@ -39,8 +39,8 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         FunctionArgumentDescriptors mandatory_args = {
-            {"Value", &isNumber<IDataType>, nullptr, "Number"},
-            {"precision", &isNativeInteger<IDataType>, &isColumnConst, "const Integer"}
+            {"Value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNumber), nullptr, "Number"},
+            {"precision", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeInteger), &isColumnConst, "const Integer"}
         };
 
         validateFunctionArgumentTypes(*this, arguments, mandatory_args, {});

src/Interpreters/Context.cpp

@@ -3270,7 +3270,7 @@ bool checkZooKeeperConfigIsLocal(const Poco::Util::AbstractConfiguration & confi
         if (startsWith(key, "node"))
         {
             String host = config.getString(config_name + "." + key + ".host");
-            if (isLocalAddress(DNSResolver::instance().resolveHost(host)))
+            if (isLocalAddress(DNSResolver::instance().resolveHostAllInOriginOrder(host).front()))
                 return true;
         }
     }

src/Interpreters/GlobalSubqueriesVisitor.h

@@ -216,7 +216,7 @@ private:
             if (enable_parallel_processing_of_joins)
             {
                 /// We don't enable parallel replicas for IN (subquery)
-                if (ast->as<ASTSubquery>())
+                if (!settings.parallel_replicas_allow_in_with_subquery && ast->as<ASTSubquery>())
                 {
                     if (settings.allow_experimental_parallel_reading_from_replicas == 1)
                     {

src/Interpreters/InterpreterCreateQuery.cpp

@@ -38,7 +38,6 @@
 #include <Storages/StorageInMemoryMetadata.h>
 #include <Storages/WindowView/StorageWindowView.h>
 #include <Storages/StorageReplicatedMergeTree.h>
-#include <Storages/BlockNumberColumn.h>
 
 #include <Interpreters/Context.h>
 #include <Interpreters/executeDDLQueryOnCluster.h>
@@ -894,24 +893,6 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat
             throw Exception(ErrorCodes::DUPLICATE_COLUMN, "Column {} already exists", backQuoteIfNeed(column.name));
     }
 
-    /// Check if _row_exists for lightweight delete column in column_lists for merge tree family.
-    if (create.storage && create.storage->engine && endsWith(create.storage->engine->name, "MergeTree"))
-    {
-        auto search = all_columns.find(LightweightDeleteDescription::FILTER_COLUMN.name);
-        if (search != all_columns.end())
-            throw Exception(ErrorCodes::ILLEGAL_COLUMN,
-                            "Cannot create table with column '{}' for *MergeTree engines because it "
-                            "is reserved for lightweight delete feature",
-                            LightweightDeleteDescription::FILTER_COLUMN.name);
-
-        auto search_block_number = all_columns.find(BlockNumberColumn::name);
-        if (search_block_number != all_columns.end())
-            throw Exception(ErrorCodes::ILLEGAL_COLUMN,
-                            "Cannot create table with column '{}' for *MergeTree engines because it "
-                            "is reserved for storing block number",
-                            BlockNumberColumn::name);
-    }
-
     const auto & settings = getContext()->getSettingsRef();
 
     /// If it's not attach and not materialized view to existing table,
@@ -924,9 +905,23 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat
     }
 }
 
+void validateVirtualColumns(const IStorage & storage)
+{
+    auto virtual_columns = storage.getVirtualsPtr();
+    for (const auto & storage_column : storage.getInMemoryMetadataPtr()->getColumns())
+    {
+        if (virtual_columns->tryGet(storage_column.name, VirtualsKind::Persistent))
+        {
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN,
+                "Cannot create table with column '{}' for {} engines because it is reserved for persistent virtual column",
+                storage_column.name, storage.getName());
+        }
+    }
+}
+
 namespace
 {
-    void checkTemporaryTableEngineName(const String& name)
+    void checkTemporaryTableEngineName(const String & name)
     {
         if (name.starts_with("Replicated") || name.starts_with("Shared") || name == "KeeperMap")
             throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables cannot be created with Replicated, Shared or KeeperMap table engines");
@@ -1509,6 +1504,16 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
         addColumnsDescriptionToCreateQueryIfNecessary(query_ptr->as<ASTCreateQuery &>(), res);
     }
 
+    validateVirtualColumns(*res);
+
+    if (!res->supportsDynamicSubcolumns() && hasDynamicSubcolumns(res->getInMemoryMetadataPtr()->getColumns()))
+    {
+        throw Exception(ErrorCodes::ILLEGAL_COLUMN,
+            "Cannot create table with column of type Object, "
+            "because storage {} doesn't support dynamic subcolumns",
+            res->getName());
+    }
+
     if (!create.attach && getContext()->getSettingsRef().database_replicated_allow_only_replicated_engine)
     {
         bool is_replicated_storage = typeid_cast<const StorageReplicatedMergeTree *>(res.get()) != nullptr;
@@ -1558,14 +1563,6 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
     /// we can safely destroy the object without a call to "shutdown", because there is guarantee
     /// that no background threads/similar resources remain after exception from "startup".
 
-    if (!res->supportsDynamicSubcolumns() && hasDynamicSubcolumns(res->getInMemoryMetadataPtr()->getColumns()))
-    {
-        throw Exception(ErrorCodes::ILLEGAL_COLUMN,
-            "Cannot create table with column of type Object, "
-            "because storage {} doesn't support dynamic subcolumns",
-            res->getName());
-    }
-
     res->startup();
     return true;
 }

src/Interpreters/InterpreterDeleteQuery.cpp

@@ -15,7 +15,6 @@
 #include <Storages/AlterCommands.h>
 #include <Storages/IStorage.h>
 #include <Storages/MutationCommands.h>
-#include <Storages/LightweightDeleteDescription.h>
 
 
 namespace DB

src/Interpreters/InterpreterDescribeQuery.cpp

@@ -123,28 +123,29 @@ BlockIO InterpreterDescribeQuery::execute()
 
 void InterpreterDescribeQuery::fillColumnsFromSubquery(const ASTTableExpression & table_expression)
 {
-    NamesAndTypesList names_and_types;
+    Block sample_block;
     auto select_query = table_expression.subquery->children.at(0);
     auto current_context = getContext();
 
     if (settings.allow_experimental_analyzer)
     {
         SelectQueryOptions select_query_options;
-        names_and_types = InterpreterSelectQueryAnalyzer(select_query, current_context, select_query_options).getSampleBlock().getNamesAndTypesList();
+        sample_block = InterpreterSelectQueryAnalyzer(select_query, current_context, select_query_options).getSampleBlock();
     }
     else
     {
-        names_and_types = InterpreterSelectWithUnionQuery::getSampleBlock(select_query, current_context).getNamesAndTypesList();
+        sample_block = InterpreterSelectWithUnionQuery::getSampleBlock(select_query, current_context);
     }
 
-    for (auto && [name, type] : names_and_types)
-        columns.emplace_back(std::move(name), std::move(type));
+    for (auto && column : sample_block)
+        columns.emplace_back(std::move(column.name), std::move(column.type));
 }
 
 void InterpreterDescribeQuery::fillColumnsFromTableFunction(const ASTTableExpression & table_expression)
 {
     auto current_context = getContext();
     TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_expression.table_function, current_context);
+
     auto column_descriptions = table_function_ptr->getActualTableStructure(getContext(), /*is_insert_query*/ true);
     for (const auto & column : column_descriptions)
         columns.emplace_back(column);
@@ -154,14 +155,16 @@ void InterpreterDescribeQuery::fillColumnsFromTableFunction(const ASTTableExpres
         auto table = table_function_ptr->execute(table_expression.table_function, getContext(), table_function_ptr->getName());
         if (table)
         {
-            for (const auto & column : table->getVirtuals())
+            auto virtuals = table->getVirtualsPtr();
+            for (const auto & column : *virtuals)
             {
                 if (!column_descriptions.has(column.name))
-                    virtual_columns.emplace_back(column.name, column.type);
+                    virtual_columns.push_back(column);
             }
         }
     }
 }
+
 void InterpreterDescribeQuery::fillColumnsFromTable(const ASTTableExpression & table_expression)
 {
     auto table_id = getContext()->resolveStorageID(table_expression.database_and_table_name);
@@ -176,10 +179,11 @@ void InterpreterDescribeQuery::fillColumnsFromTable(const ASTTableExpression & t
 
     if (settings.describe_include_virtual_columns)
     {
-        for (const auto & column : table->getVirtuals())
+        auto virtuals = table->getVirtualsPtr();
+        for (const auto & column : *virtuals)
         {
             if (!column_descriptions.has(column.name))
-                virtual_columns.emplace_back(column.name, column.type);
+                virtual_columns.push_back(column);
         }
     }
 

src/Interpreters/InterpreterInsertQuery.cpp

@@ -136,7 +136,7 @@ Block InterpreterInsertQuery::getSampleBlock(
         if (auto * window_view = dynamic_cast<StorageWindowView *>(table.get()))
             return window_view->getInputHeader();
         else if (no_destination)
-            return metadata_snapshot->getSampleBlockWithVirtuals(table->getVirtuals());
+            return metadata_snapshot->getSampleBlockWithVirtuals(table->getVirtualsList());
         else
             return metadata_snapshot->getSampleBlockNonMaterialized();
     }

src/Interpreters/InterpreterShowColumnsQuery.cpp

@@ -107,7 +107,7 @@ SELECT
     '' AS extra )";
 
     // TODO Interpret query.extended. It is supposed to show internal/virtual columns. Need to fetch virtual column names, see
-    // IStorage::getVirtuals(). We can't easily do that via SQL.
+    // IStorage::getVirtualsList(). We can't easily do that via SQL.
 
     if (query.full)
     {

src/Interpreters/JoinedTables.cpp

@@ -272,7 +272,7 @@ void JoinedTables::makeFakeTable(StoragePtr storage, const StorageMetadataPtr &
         auto & table = tables_with_columns.back();
         table.addHiddenColumns(storage_columns.getMaterialized());
         table.addHiddenColumns(storage_columns.getAliases());
-        table.addHiddenColumns(storage->getVirtuals());
+        table.addHiddenColumns(storage->getVirtualsList());
     }
     else
         tables_with_columns.emplace_back(DatabaseAndTableWithAlias{}, source_header.getNamesAndTypesList());

src/Interpreters/MutationsInterpreter.cpp

@@ -7,7 +7,7 @@
 #include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/MergeTree/StorageFromMergeTreeDataPart.h>
 #include <Storages/StorageMergeTree.h>
-#include <Storages/BlockNumberColumn.h>
+#include <Storages/MergeTree/MergeTreeVirtualColumns.h>
 #include <Processors/Transforms/FilterTransform.h>
 #include <Processors/Transforms/ExpressionTransform.h>
 #include <Processors/Transforms/CreatingSetsTransform.h>
@@ -31,7 +31,6 @@
 #include <Processors/QueryPlan/CreatingSetsStep.h>
 #include <DataTypes/NestedUtils.h>
 #include <Interpreters/PreparedSets.h>
-#include <Storages/LightweightDeleteDescription.h>
 #include <Storages/MergeTree/MergeTreeSequentialSource.h>
 #include <Processors/Sources/ThrowingExceptionSource.h>
 #include <Analyzer/QueryTreeBuilder.h>
@@ -265,7 +264,7 @@ MutationCommand createCommandToApplyDeletedMask(const MutationCommand & command)
     alter_command->partition = alter_command->children.emplace_back(command.partition).get();
 
     auto row_exists_predicate = makeASTFunction("equals",
-        std::make_shared<ASTIdentifier>(LightweightDeleteDescription::FILTER_COLUMN.name),
+        std::make_shared<ASTIdentifier>(RowExistsColumn::name),
         std::make_shared<ASTLiteral>(Field(0)));
 
     if (command.predicate)
@@ -350,7 +349,8 @@ bool MutationsInterpreter::Source::isCompactPart() const
 static Names getAvailableColumnsWithVirtuals(StorageMetadataPtr metadata_snapshot, const IStorage & storage)
 {
     auto all_columns = metadata_snapshot->getColumns().getNamesOfPhysical();
-    for (const auto & column : storage.getVirtuals())
+    auto virtuals = storage.getVirtualsPtr();
+    for (const auto & column : *virtuals)
         all_columns.push_back(column.name);
     return all_columns;
 }
@@ -435,60 +435,54 @@ static NameSet getKeyColumns(const MutationsInterpreter::Source & source, const
 
 static void validateUpdateColumns(
     const MutationsInterpreter::Source & source,
-    const StorageMetadataPtr & metadata_snapshot, const NameSet & updated_columns,
-    const std::unordered_map<String, Names> & column_to_affected_materialized)
+    const StorageMetadataPtr & metadata_snapshot,
+    const NameSet & updated_columns,
+    const std::unordered_map<String, Names> & column_to_affected_materialized,
+    const ContextPtr & context)
 {
+    auto storage_snapshot = source.getStorageSnapshot(metadata_snapshot, context);
     NameSet key_columns = getKeyColumns(source, metadata_snapshot);
 
-    for (const String & column_name : updated_columns)
+    const auto & storage_columns = storage_snapshot->metadata->getColumns();
+    const auto & virtual_columns = *storage_snapshot->virtual_columns;
+
+    for (const auto & column_name : updated_columns)
     {
-        auto found = false;
-        for (const auto & col : metadata_snapshot->getColumns().getOrdinary())
-        {
-            if (col.name == column_name)
-            {
-                found = true;
-                break;
-            }
-        }
-
-        /// Allow to override value of lightweight delete filter virtual column
-        if (!found && column_name == LightweightDeleteDescription::FILTER_COLUMN.name)
-        {
-            if (!source.supportsLightweightDelete())
-                throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Lightweight delete is not supported for table");
-            found = true;
-        }
-
-        /// Dont allow to override value of block number virtual column
-        if (!found && column_name == BlockNumberColumn::name)
-        {
-            throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Update is not supported for virtual column {} ", backQuote(column_name));
-        }
-
-        if (!found)
-        {
-            for (const auto & col : metadata_snapshot->getColumns().getMaterialized())
-            {
-                if (col.name == column_name)
-                    throw Exception(ErrorCodes::CANNOT_UPDATE_COLUMN, "Cannot UPDATE materialized column {}", backQuote(column_name));
-            }
-
-            throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "There is no column {} in table", backQuote(column_name));
-        }
-
         if (key_columns.contains(column_name))
             throw Exception(ErrorCodes::CANNOT_UPDATE_COLUMN, "Cannot UPDATE key column {}", backQuote(column_name));
 
+        if (storage_columns.tryGetColumn(GetColumnsOptions::Materialized, column_name))
+            throw Exception(ErrorCodes::CANNOT_UPDATE_COLUMN, "Cannot UPDATE materialized column {}", backQuote(column_name));
+
         auto materialized_it = column_to_affected_materialized.find(column_name);
         if (materialized_it != column_to_affected_materialized.end())
         {
-            for (const String & materialized : materialized_it->second)
+            for (const auto & materialized : materialized_it->second)
             {
                 if (key_columns.contains(materialized))
+                {
                     throw Exception(ErrorCodes::CANNOT_UPDATE_COLUMN,
                                     "Updated column {} affects MATERIALIZED column {}, which is a key column. "
                                     "Cannot UPDATE it.", backQuote(column_name), backQuote(materialized));
+                }
+            }
+        }
+
+        if (!storage_columns.tryGetColumn(GetColumnsOptions::Ordinary, column_name))
+        {
+            /// Allow to override value of lightweight delete filter virtual column
+            if (column_name == RowExistsColumn::name)
+            {
+                if (!source.supportsLightweightDelete())
+                    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Lightweight delete is not supported for table");
+            }
+            else if (virtual_columns.tryGet(column_name))
+            {
+                throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Update is not supported for virtual column {} ", backQuote(column_name));
+            }
+            else
+            {
+                throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "There is no column {} in table", backQuote(column_name));
             }
         }
     }
@@ -546,8 +540,8 @@ void MutationsInterpreter::prepare(bool dry_run)
     /// Add _row_exists column if it is physically present in the part
     if (source.hasLightweightDeleteMask())
     {
-        all_columns.push_back({LightweightDeleteDescription::FILTER_COLUMN});
-        available_columns_set.insert(LightweightDeleteDescription::FILTER_COLUMN.name);
+        all_columns.emplace_back(RowExistsColumn::name, RowExistsColumn::type);
+        available_columns_set.insert(RowExistsColumn::name);
     }
 
     NameSet updated_columns;
@@ -563,9 +557,7 @@ void MutationsInterpreter::prepare(bool dry_run)
 
         for (const auto & [name, _] : command.column_to_update_expression)
         {
-            if (!available_columns_set.contains(name)
-                && name != LightweightDeleteDescription::FILTER_COLUMN.name
-                && name != BlockNumberColumn::name)
+            if (!available_columns_set.contains(name) && name != RowExistsColumn::name)
                 throw Exception(ErrorCodes::THERE_IS_NO_COLUMN,
                     "Column {} is updated but not requested to read", name);
 
@@ -590,7 +582,7 @@ void MutationsInterpreter::prepare(bool dry_run)
             }
         }
 
-        validateUpdateColumns(source, metadata_snapshot, updated_columns, column_to_affected_materialized);
+        validateUpdateColumns(source, metadata_snapshot, updated_columns, column_to_affected_materialized, context);
     }
 
     StorageInMemoryMetadata::HasDependencyCallback has_dependency =
@@ -666,15 +658,11 @@ void MutationsInterpreter::prepare(bool dry_run)
                 {
                     type = physical_column->type;
                 }
-                else if (column_name == LightweightDeleteDescription::FILTER_COLUMN.name)
+                else if (column_name == RowExistsColumn::name)
                 {
-                    type = LightweightDeleteDescription::FILTER_COLUMN.type;
+                    type = RowExistsColumn::type;
                     deleted_mask_updated = true;
                 }
-                else if (column_name == BlockNumberColumn::name)
-                {
-                    type = BlockNumberColumn::type;
-                }
                 else
                 {
                     throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown column {}", column_name);
@@ -1028,7 +1016,7 @@ void MutationsInterpreter::prepareMutationStages(std::vector<Stage> & prepared_s
 
     /// Add _row_exists column if it is present in the part
     if (source.hasLightweightDeleteMask() || deleted_mask_updated)
-        all_columns.push_back(LightweightDeleteDescription::FILTER_COLUMN);
+        all_columns.emplace_back(RowExistsColumn::name, RowExistsColumn::type);
 
     bool has_filters = false;
     /// Next, for each stage calculate columns changed by this and previous stages.
@@ -1038,7 +1026,7 @@ void MutationsInterpreter::prepareMutationStages(std::vector<Stage> & prepared_s
         {
             for (const auto & column : all_columns)
             {
-                if (column.name == LightweightDeleteDescription::FILTER_COLUMN.name && !deleted_mask_updated)
+                if (column.name == RowExistsColumn::name && !deleted_mask_updated)
                     continue;
 
                 prepared_stages[i].output_columns.insert(column.name);
@@ -1057,7 +1045,7 @@ void MutationsInterpreter::prepareMutationStages(std::vector<Stage> & prepared_s
             /// and so it is not in the list of AllPhysical columns.
             for (const auto & [column_name, _] : prepared_stages[i].column_to_updated)
             {
-                if (column_name == LightweightDeleteDescription::FILTER_COLUMN.name && has_filters && !deleted_mask_updated)
+                if (column_name == RowExistsColumn::name && has_filters && !deleted_mask_updated)
                     continue;
 
                 prepared_stages[i].output_columns.insert(column_name);
@@ -1148,93 +1136,6 @@ void MutationsInterpreter::prepareMutationStages(std::vector<Stage> & prepared_s
     }
 }
 
-/// This structure re-implements adding virtual columns while reading from MergeTree part.
-/// It would be good to unify it with IMergeTreeSelectAlgorithm.
-struct VirtualColumns
-{
-    struct ColumnAndPosition
-    {
-        ColumnWithTypeAndName column;
-        size_t position;
-    };
-
-    using Columns = std::vector<ColumnAndPosition>;
-
-    Columns virtuals;
-    Names columns_to_read;
-
-    VirtualColumns(Names required_columns, const MergeTreeData::DataPartPtr & part) : columns_to_read(std::move(required_columns))
-    {
-        for (size_t i = 0; i < columns_to_read.size(); ++i)
-        {
-            if (columns_to_read[i] == LightweightDeleteDescription::FILTER_COLUMN.name)
-            {
-                if (!part->getColumns().contains(LightweightDeleteDescription::FILTER_COLUMN.name))
-                {
-                    ColumnWithTypeAndName mask_column;
-                    mask_column.type = LightweightDeleteDescription::FILTER_COLUMN.type;
-                    mask_column.column = mask_column.type->createColumnConst(0, 1);
-                    mask_column.name = std::move(columns_to_read[i]);
-
-                    virtuals.emplace_back(ColumnAndPosition{.column = std::move(mask_column), .position = i});
-                }
-            }
-            else if (columns_to_read[i] == "_partition_id")
-            {
-                ColumnWithTypeAndName column;
-                column.type = std::make_shared<DataTypeString>();
-                column.column = column.type->createColumnConst(0, part->info.partition_id);
-                column.name = std::move(columns_to_read[i]);
-
-                virtuals.emplace_back(ColumnAndPosition{.column = std::move(column), .position = i});
-            }
-            else if (columns_to_read[i] == BlockNumberColumn::name)
-            {
-                if (!part->getColumns().contains(BlockNumberColumn::name))
-                {
-                    ColumnWithTypeAndName block_number_column;
-                    block_number_column.type = BlockNumberColumn::type;
-                    block_number_column.column = block_number_column.type->createColumnConst(0, part->info.min_block);
-                    block_number_column.name = std::move(columns_to_read[i]);
-
-                    virtuals.emplace_back(ColumnAndPosition{.column = std::move(block_number_column), .position = i});
-                }
-            }
-        }
-
-        if (!virtuals.empty())
-        {
-            Names columns_no_virtuals;
-            columns_no_virtuals.reserve(columns_to_read.size());
-            size_t next_virtual = 0;
-            for (size_t i = 0; i < columns_to_read.size(); ++i)
-            {
-                if (next_virtual < virtuals.size() && i == virtuals[next_virtual].position)
-                    ++next_virtual;
-                else
-                    columns_no_virtuals.emplace_back(std::move(columns_to_read[i]));
-            }
-
-            columns_to_read.swap(columns_no_virtuals);
-        }
-    }
-
-    void addVirtuals(QueryPlan & plan)
-    {
-        auto dag = std::make_unique<ActionsDAG>(plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
-
-        for (auto & column : virtuals)
-        {
-            const auto & adding_const = dag->addColumn(std::move(column.column));
-            auto & outputs = dag->getOutputs();
-            outputs.insert(outputs.begin() + column.position, &adding_const);
-        }
-
-        auto step = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), std::move(dag));
-        plan.addStep(std::move(step));
-    }
-};
-
 void MutationsInterpreter::Source::read(
     Stage & first_stage,
     QueryPlan & plan,
@@ -1277,16 +1178,12 @@ void MutationsInterpreter::Source::read(
             filter = ActionsDAG::buildFilterActionsDAG(nodes);
         }
 
-        VirtualColumns virtual_columns(std::move(required_columns), part);
-
         createReadFromPartStep(
             MergeTreeSequentialSourceType::Mutation,
-            plan, *data, storage_snapshot, part,
-            std::move(virtual_columns.columns_to_read),
+            plan, *data, storage_snapshot,
+            part, required_columns,
             apply_deleted_mask_, filter, context_,
             getLogger("MutationsInterpreter"));
-
-        virtual_columns.addVirtuals(plan);
     }
     else
     {

src/Interpreters/TreeRewriter.cpp

@@ -56,6 +56,7 @@
 #include <Storages/IStorage.h>
 #include <Storages/StorageJoin.h>
 #include <Common/checkStackSize.h>
+#include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/StorageView.h>
 
 #include <AggregateFunctions/AggregateFunctionFactory.h>
@@ -990,8 +991,7 @@ void TreeRewriterResult::collectSourceColumns(bool add_special)
     {
         auto options = GetColumnsOptions(add_special ? GetColumnsOptions::All : GetColumnsOptions::AllPhysical);
         options.withExtendedObjects();
-        if (storage->supportsSubcolumns())
-            options.withSubcolumns();
+        options.withSubcolumns(storage->supportsSubcolumns());
 
         auto columns_from_storage = storage_snapshot->getColumns(options);
 
@@ -1001,8 +1001,7 @@ void TreeRewriterResult::collectSourceColumns(bool add_special)
             source_columns.insert(source_columns.end(), columns_from_storage.begin(), columns_from_storage.end());
 
         auto metadata_snapshot = storage->getInMemoryMetadataPtr();
-        auto metadata_column_descriptions = metadata_snapshot->getColumns();
-        source_columns_ordinary = metadata_column_descriptions.getOrdinary();
+        source_columns_ordinary = metadata_snapshot->getColumns().getOrdinary();
     }
 
     source_columns_set = removeDuplicateColumns(source_columns);
@@ -1109,16 +1108,16 @@ bool TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select
         const auto & partition_desc = storage_snapshot->metadata->getPartitionKey();
         if (partition_desc.expression)
         {
-            auto partition_source_columns = partition_desc.expression->getRequiredColumns();
-            partition_source_columns.push_back("_part");
-            partition_source_columns.push_back("_partition_id");
-            partition_source_columns.push_back("_part_uuid");
-            partition_source_columns.push_back("_partition_value");
+            auto partition_columns = partition_desc.expression->getRequiredColumns();
+            NameSet partition_columns_set(partition_columns.begin(), partition_columns.end());
+
+            const auto & parititon_virtuals = MergeTreeData::virtuals_useful_for_filter;
+            partition_columns_set.insert(parititon_virtuals.begin(), parititon_virtuals.end());
+
             optimize_trivial_count = true;
             for (const auto & required_column : required)
             {
-                if (std::find(partition_source_columns.begin(), partition_source_columns.end(), required_column)
-                    == partition_source_columns.end())
+                if (!partition_columns_set.contains(required_column))
                 {
                     optimize_trivial_count = false;
                     break;
@@ -1129,7 +1128,7 @@ bool TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select
 
     NameSet unknown_required_source_columns = required;
 
-    for (NamesAndTypesList::iterator it = source_columns.begin(); it != source_columns.end();)
+    for (auto it = source_columns.begin(); it != source_columns.end();)
     {
         const String & column_name = it->name;
         unknown_required_source_columns.erase(column_name);
@@ -1143,32 +1142,23 @@ bool TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select
     has_virtual_shard_num = false;
     /// If there are virtual columns among the unknown columns. Remove them from the list of unknown and add
     /// in columns list, so that when further processing they are also considered.
-    if (storage)
+    if (storage_snapshot)
     {
-        const auto storage_virtuals = storage->getVirtuals();
+        const auto & virtuals = storage_snapshot->virtual_columns;
         for (auto it = unknown_required_source_columns.begin(); it != unknown_required_source_columns.end();)
         {
-            auto column = storage_virtuals.tryGetByName(*it);
-            if (column)
+            if (auto column = virtuals->tryGet(*it))
             {
                 source_columns.push_back(*column);
                 it = unknown_required_source_columns.erase(it);
             }
             else
-                ++it;
-        }
-
-        if (is_remote_storage)
-        {
-            for (const auto & name_type : storage_virtuals)
             {
-                if (name_type.name == "_shard_num" && storage->isVirtualColumn("_shard_num", storage_snapshot->getMetadataForQuery()))
-                {
-                    has_virtual_shard_num = true;
-                    break;
-                }
+                ++it;
             }
         }
+
+        has_virtual_shard_num = is_remote_storage && storage->isVirtualColumn("_shard_num", storage_snapshot->getMetadataForQuery()) && virtuals->has("_shard_num");
     }
 
     /// Collect missed object subcolumns

src/Interpreters/getTableExpressions.cpp

@@ -99,7 +99,7 @@ static NamesAndTypesList getColumnsFromTableExpression(
         names_and_type_list = columns.getOrdinary();
         materialized = columns.getMaterialized();
         aliases = columns.getAliases();
-        virtuals = function_storage->getVirtuals();
+        virtuals = function_storage->getVirtualsList();
     }
     else if (table_expression.database_and_table_name)
     {
@@ -110,7 +110,7 @@ static NamesAndTypesList getColumnsFromTableExpression(
         names_and_type_list = columns.getOrdinary();
         materialized = columns.getMaterialized();
         aliases = columns.getAliases();
-        virtuals = table->getVirtuals();
+        virtuals = table->getVirtualsList();
     }
 
     return names_and_type_list;

src/Interpreters/inplaceBlockConversions.cpp

@@ -20,7 +20,6 @@
 #include <Columns/ColumnArray.h>
 #include <DataTypes/DataTypeArray.h>
 #include <Storages/StorageInMemoryMetadata.h>
-#include <Storages/BlockNumberColumn.h>
 
 
 namespace DB
@@ -280,7 +279,7 @@ void fillMissingColumns(
     const NamesAndTypesList & requested_columns,
     const NamesAndTypesList & available_columns,
     const NameSet & partially_read_columns,
-    StorageMetadataPtr metadata_snapshot, size_t block_number)
+    StorageMetadataPtr metadata_snapshot)
 {
     size_t num_columns = requested_columns.size();
     if (num_columns != res_columns.size())
@@ -359,14 +358,9 @@ void fillMissingColumns(
         }
         else
         {
-            if (requested_column->name == BlockNumberColumn::name)
-                res_columns[i] = type->createColumnConst(num_rows, block_number)->convertToFullColumnIfConst();
-            else
-                /// We must turn a constant column into a full column because the interpreter could infer
-                /// that it is constant everywhere but in some blocks (from other parts) it can be a full column.
-                res_columns[i] = type->createColumnConstWithDefaultValue(num_rows)->convertToFullColumnIfConst();
-
-
+            /// We must turn a constant column into a full column because the interpreter could infer
+            /// that it is constant everywhere but in some blocks (from other parts) it can be a full column.
+            res_columns[i] = type->createColumnConstWithDefaultValue(num_rows)->convertToFullColumnIfConst();
         }
     }
 }

src/Interpreters/inplaceBlockConversions.h

@@ -46,6 +46,6 @@ void fillMissingColumns(
     const NamesAndTypesList & requested_columns,
     const NamesAndTypesList & available_columns,
     const NameSet & partially_read_columns,
-    StorageMetadataPtr metadata_snapshot, size_t block_number = 0);
+    StorageMetadataPtr metadata_snapshot);
 
 }

src/Interpreters/processColumnTransformers.cpp

@@ -32,7 +32,7 @@ ASTPtr processColumnTransformers(
 
     tables_with_columns[0].addHiddenColumns(columns.getMaterialized());
     tables_with_columns[0].addHiddenColumns(columns.getAliases());
-    tables_with_columns[0].addHiddenColumns(table->getVirtuals());
+    tables_with_columns[0].addHiddenColumns(table->getVirtualsList());
 
     NameSet source_columns_set;
     for (const auto & identifier : query_columns->children)

src/Parsers/ASTFunction.cpp

@@ -18,6 +18,7 @@
 #include <Parsers/ASTSubquery.h>
 #include <Parsers/queryToString.h>
 #include <Parsers/ASTSetQuery.h>
+#include <Parsers/FunctionSecretArgumentsFinderAST.h>
 #include <Core/QualifiedTableName.h>
 
 #include <boost/algorithm/string.hpp>
@@ -36,508 +37,6 @@ namespace ErrorCodes
 }
 
 
-namespace
-{
-    /// Finds arguments of a specified function which should not be displayed for most users for security reasons.
-    /// That involves passwords and secret keys.
-    class FunctionSecretArgumentsFinder
-    {
-    public:
-        explicit FunctionSecretArgumentsFinder(const ASTFunction & function_) : function(function_)
-        {
-            if (!function.arguments)
-                return;
-
-            const auto * expr_list = function.arguments->as<ASTExpressionList>();
-            if (!expr_list)
-                return;
-
-            arguments = &expr_list->children;
-            switch (function.kind)
-            {
-                case ASTFunction::Kind::ORDINARY_FUNCTION: findOrdinaryFunctionSecretArguments(); break;
-                case ASTFunction::Kind::WINDOW_FUNCTION: break;
-                case ASTFunction::Kind::LAMBDA_FUNCTION:  break;
-                case ASTFunction::Kind::TABLE_ENGINE: findTableEngineSecretArguments(); break;
-                case ASTFunction::Kind::DATABASE_ENGINE: findDatabaseEngineSecretArguments(); break;
-                case ASTFunction::Kind::BACKUP_NAME: findBackupNameSecretArguments(); break;
-            }
-        }
-
-        struct Result
-        {
-            /// Result constructed by default means no arguments will be hidden.
-            size_t start = static_cast<size_t>(-1);
-            size_t count = 0; /// Mostly it's either 0 or 1. There are only a few cases where `count` can be greater than 1 (e.g. see `encrypt`).
-                              /// In all known cases secret arguments are consecutive
-            bool are_named = false; /// Arguments like `password = 'password'` are considered as named arguments.
-            /// E.g. "headers" in `url('..', headers('foo' = '[HIDDEN]'))`
-            std::vector<std::string> nested_maps;
-
-            bool hasSecrets() const
-            {
-                return count != 0 || !nested_maps.empty();
-            }
-        };
-
-        Result getResult() const { return result; }
-
-    private:
-        const ASTFunction & function;
-        const ASTs * arguments = nullptr;
-        Result result;
-
-        void markSecretArgument(size_t index, bool argument_is_named = false)
-        {
-            if (index >= arguments->size())
-                return;
-            if (!result.count)
-            {
-                result.start = index;
-                result.are_named = argument_is_named;
-            }
-            chassert(index >= result.start); /// We always check arguments consecutively
-            result.count = index + 1 - result.start;
-            if (!argument_is_named)
-                result.are_named = false;
-        }
-
-        void findOrdinaryFunctionSecretArguments()
-        {
-            if ((function.name == "mysql") || (function.name == "postgresql") || (function.name == "mongodb"))
-            {
-                /// mysql('host:port', 'database', 'table', 'user', 'password', ...)
-                /// postgresql('host:port', 'database', 'table', 'user', 'password', ...)
-                /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...)
-                findMySQLFunctionSecretArguments();
-            }
-            else if ((function.name == "s3") || (function.name == "cosn") || (function.name == "oss") ||
-                     (function.name == "deltaLake") || (function.name == "hudi") || (function.name == "iceberg"))
-            {
-                /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
-                findS3FunctionSecretArguments(/* is_cluster_function= */ false);
-            }
-            else if (function.name == "s3Cluster")
-            {
-                /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...)
-                findS3FunctionSecretArguments(/* is_cluster_function= */ true);
-            }
-            else if ((function.name == "remote") || (function.name == "remoteSecure"))
-            {
-                /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...)
-                findRemoteFunctionSecretArguments();
-            }
-            else if ((function.name == "encrypt") || (function.name == "decrypt") ||
-                     (function.name == "aes_encrypt_mysql") || (function.name == "aes_decrypt_mysql") ||
-                     (function.name == "tryDecrypt"))
-            {
-                /// encrypt('mode', 'plaintext', 'key' [, iv, aad])
-                findEncryptionFunctionSecretArguments();
-            }
-            else if (function.name == "url")
-            {
-                findURLSecretArguments();
-            }
-        }
-
-        void findMySQLFunctionSecretArguments()
-        {
-            if (isNamedCollectionName(0))
-            {
-                /// mysql(named_collection, ..., password = 'password', ...)
-                findSecretNamedArgument("password", 1);
-            }
-            else
-            {
-                /// mysql('host:port', 'database', 'table', 'user', 'password', ...)
-                markSecretArgument(4);
-            }
-        }
-
-        /// Returns the number of arguments excluding "headers" and "extra_credentials" (which should
-        /// always be at the end). Marks "headers" as secret, if found.
-        size_t excludeS3OrURLNestedMaps()
-        {
-            size_t count = arguments->size();
-            while (count > 0)
-            {
-                const ASTFunction * f = arguments->at(count - 1)->as<ASTFunction>();
-                if (!f)
-                    break;
-                if (f->name == "headers")
-                    result.nested_maps.push_back(f->name);
-                else if (f->name != "extra_credentials")
-                    break;
-                count -= 1;
-            }
-            return count;
-        }
-
-        void findS3FunctionSecretArguments(bool is_cluster_function)
-        {
-            /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument.
-            size_t url_arg_idx = is_cluster_function ? 1 : 0;
-
-            if (!is_cluster_function && isNamedCollectionName(0))
-            {
-                /// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...)
-                findSecretNamedArgument("secret_access_key", 1);
-                return;
-            }
-
-            /// We should check other arguments first because we don't need to do any replacement in case of
-            /// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
-            /// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
-            size_t count = excludeS3OrURLNestedMaps();
-            if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4))
-            {
-                String second_arg;
-                if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg))
-                {
-                    if (boost::iequals(second_arg, "NOSIGN"))
-                        return; /// The argument after 'url' is "NOSIGN".
-
-                    if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg))
-                        return; /// The argument after 'url' is a format: s3('url', 'format', ...)
-                }
-            }
-
-            /// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures:
-            /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
-            /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
-            if (url_arg_idx + 2 < count)
-                markSecretArgument(url_arg_idx + 2);
-        }
-
-        void findURLSecretArguments()
-        {
-            if (!isNamedCollectionName(0))
-                excludeS3OrURLNestedMaps();
-        }
-
-        bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const
-        {
-            if (arg_idx >= arguments->size())
-                return false;
-
-            return tryGetStringFromArgument(*(*arguments)[arg_idx], res, allow_identifier);
-        }
-
-        static bool tryGetStringFromArgument(const IAST & argument, String * res, bool allow_identifier = true)
-        {
-            if (const auto * literal = argument.as<ASTLiteral>())
-            {
-                if (literal->value.getType() != Field::Types::String)
-                    return false;
-                if (res)
-                    *res = literal->value.safeGet<String>();
-                return true;
-            }
-
-            if (allow_identifier)
-            {
-                if (const auto * id = argument.as<ASTIdentifier>())
-                {
-                    if (res)
-                        *res = id->name();
-                    return true;
-                }
-            }
-
-            return false;
-        }
-
-        void findRemoteFunctionSecretArguments()
-        {
-            if (isNamedCollectionName(0))
-            {
-                /// remote(named_collection, ..., password = 'password', ...)
-                findSecretNamedArgument("password", 1);
-                return;
-            }
-
-            /// We're going to replace 'password' with '[HIDDEN'] for the following signatures:
-            /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key])
-            /// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key])
-            /// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key])
-
-            /// But we should check the number of arguments first because we don't need to do any replacements in case of
-            /// remote('addresses_expr', db.table)
-            if (arguments->size() < 3)
-                return;
-
-            size_t arg_num = 1;
-
-            /// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'.
-            const auto * table_function = (*arguments)[arg_num]->as<ASTFunction>();
-            if (table_function && KnownTableFunctionNames::instance().exists(table_function->name))
-            {
-                ++arg_num;
-            }
-            else
-            {
-                std::optional<String> database;
-                std::optional<QualifiedTableName> qualified_table_name;
-                if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name))
-                {
-                    /// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'.
-                    /// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user'
-                    /// before the argument 'password'. So it's safer to wipe two arguments just in case.
-                    /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
-                    /// before wiping it (because the `password` argument is always a literal string).
-                    if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false))
-                    {
-                        /// Wipe either `password` or `user`.
-                        markSecretArgument(arg_num + 2);
-                    }
-                    if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false))
-                    {
-                        /// Wipe either `password` or `sharding_key`.
-                        markSecretArgument(arg_num + 3);
-                    }
-                    return;
-                }
-
-                /// Skip the current argument (which is either a database name or a qualified table name).
-                ++arg_num;
-                if (database)
-                {
-                    /// Skip the 'table' argument if the previous argument was a database name.
-                    ++arg_num;
-                }
-            }
-
-            /// Skip username.
-            ++arg_num;
-
-            /// Do our replacement:
-            /// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...)
-            /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
-            /// before wiping it (because the `password` argument is always a literal string).
-            bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false);
-            if (can_be_password)
-                markSecretArgument(arg_num);
-        }
-
-        /// Tries to get either a database name or a qualified table name from an argument.
-        /// Empty string is also allowed (it means the default database).
-        /// The function is used by findRemoteFunctionSecretArguments() to determine how many arguments to skip before a password.
-        bool tryGetDatabaseNameOrQualifiedTableName(
-            size_t arg_idx,
-            std::optional<String> & res_database,
-            std::optional<QualifiedTableName> & res_qualified_table_name) const
-        {
-            res_database.reset();
-            res_qualified_table_name.reset();
-
-            String str;
-            if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true))
-                return false;
-
-            if (str.empty())
-            {
-                res_database = "";
-                return true;
-            }
-
-            auto qualified_table_name = QualifiedTableName::tryParseFromString(str);
-            if (!qualified_table_name)
-                return false;
-
-            if (qualified_table_name->database.empty())
-                res_database = std::move(qualified_table_name->table);
-            else
-                res_qualified_table_name = std::move(qualified_table_name);
-            return true;
-        }
-
-        void findEncryptionFunctionSecretArguments()
-        {
-            if (arguments->empty())
-                return;
-
-            /// We replace all arguments after 'mode' with '[HIDDEN]':
-            /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]')
-            result.start = 1;
-            result.count = arguments->size() - 1;
-        }
-
-        void findTableEngineSecretArguments()
-        {
-            const String & engine_name = function.name;
-            if (engine_name == "ExternalDistributed")
-            {
-                /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password')
-                findExternalDistributedTableEngineSecretArguments();
-            }
-            else if ((engine_name == "MySQL") || (engine_name == "PostgreSQL") ||
-                     (engine_name == "MaterializedPostgreSQL") || (engine_name == "MongoDB"))
-            {
-                /// MySQL('host:port', 'database', 'table', 'user', 'password', ...)
-                /// PostgreSQL('host:port', 'database', 'table', 'user', 'password', ...)
-                /// MaterializedPostgreSQL('host:port', 'database', 'table', 'user', 'password', ...)
-                /// MongoDB('host:port', 'database', 'collection', 'user', 'password', ...)
-                findMySQLFunctionSecretArguments();
-            }
-            else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS") ||
-                     (engine_name == "DeltaLake") || (engine_name == "Hudi") || (engine_name == "Iceberg") || (engine_name == "S3Queue"))
-            {
-                /// S3('url', ['aws_access_key_id', 'aws_secret_access_key',] ...)
-                findS3TableEngineSecretArguments();
-            }
-            else if (engine_name == "URL")
-            {
-                findURLSecretArguments();
-            }
-        }
-
-        void findExternalDistributedTableEngineSecretArguments()
-        {
-            if (isNamedCollectionName(1))
-            {
-                /// ExternalDistributed('engine', named_collection, ..., password = 'password', ...)
-                findSecretNamedArgument("password", 2);
-            }
-            else
-            {
-                /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password')
-                markSecretArgument(5);
-            }
-        }
-
-        void findS3TableEngineSecretArguments()
-        {
-            if (isNamedCollectionName(0))
-            {
-                /// S3(named_collection, ..., secret_access_key = 'secret_access_key')
-                findSecretNamedArgument("secret_access_key", 1);
-                return;
-            }
-
-            /// We should check other arguments first because we don't need to do any replacement in case of
-            /// S3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
-            /// S3('url', 'format', 'compression' [, extra_credentials(..)] [, headers(..)])
-            size_t count = excludeS3OrURLNestedMaps();
-            if ((3 <= count) && (count <= 4))
-            {
-                String second_arg;
-                if (tryGetStringFromArgument(1, &second_arg))
-                {
-                    if (boost::iequals(second_arg, "NOSIGN"))
-                        return; /// The argument after 'url' is "NOSIGN".
-
-                    if (count == 3)
-                    {
-                        if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg))
-                            return; /// The argument after 'url' is a format: S3('url', 'format', ...)
-                    }
-                }
-            }
-
-            /// We replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures:
-            /// S3('url', 'aws_access_key_id', 'aws_secret_access_key')
-            /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format')
-            /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
-            if (2 < count)
-                markSecretArgument(2);
-        }
-
-        void findDatabaseEngineSecretArguments()
-        {
-            const String & engine_name = function.name;
-            if ((engine_name == "MySQL") || (engine_name == "MaterializeMySQL") ||
-                (engine_name == "MaterializedMySQL") || (engine_name == "PostgreSQL") ||
-                (engine_name == "MaterializedPostgreSQL"))
-            {
-                /// MySQL('host:port', 'database', 'user', 'password')
-                /// PostgreSQL('host:port', 'database', 'user', 'password')
-                findMySQLDatabaseSecretArguments();
-            }
-            else if (engine_name == "S3")
-            {
-                /// S3('url', 'access_key_id', 'secret_access_key')
-                findS3DatabaseSecretArguments();
-            }
-        }
-
-        void findMySQLDatabaseSecretArguments()
-        {
-            if (isNamedCollectionName(0))
-            {
-                /// MySQL(named_collection, ..., password = 'password', ...)
-                findSecretNamedArgument("password", 1);
-            }
-            else
-            {
-                /// MySQL('host:port', 'database', 'user', 'password')
-                markSecretArgument(3);
-            }
-        }
-
-        void findS3DatabaseSecretArguments()
-        {
-            if (isNamedCollectionName(0))
-            {
-                /// S3(named_collection, ..., secret_access_key = 'password', ...)
-                findSecretNamedArgument("secret_access_key", 1);
-            }
-            else
-            {
-                /// S3('url', 'access_key_id', 'secret_access_key')
-                markSecretArgument(2);
-            }
-        }
-
-        void findBackupNameSecretArguments()
-        {
-            const String & engine_name = function.name;
-            if (engine_name == "S3")
-            {
-                /// BACKUP ... TO S3(url, [aws_access_key_id, aws_secret_access_key])
-                markSecretArgument(2);
-            }
-        }
-
-        /// Whether a specified argument can be the name of a named collection?
-        bool isNamedCollectionName(size_t arg_idx) const
-        {
-            if (arguments->size() <= arg_idx)
-                return false;
-
-            const auto * identifier = (*arguments)[arg_idx]->as<ASTIdentifier>();
-            return identifier != nullptr;
-        }
-
-        /// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified.
-        void findSecretNamedArgument(const std::string_view & key, size_t start = 0)
-        {
-            for (size_t i = start; i < arguments->size(); ++i)
-            {
-                const auto & argument = (*arguments)[i];
-                const auto * equals_func = argument->as<ASTFunction>();
-                if (!equals_func || (equals_func->name != "equals"))
-                    continue;
-
-                const auto * expr_list = equals_func->arguments->as<ASTExpressionList>();
-                if (!expr_list)
-                    continue;
-
-                const auto & equal_args = expr_list->children;
-                if (equal_args.size() != 2)
-                    continue;
-
-                String found_key;
-                if (!tryGetStringFromArgument(*equal_args[0], &found_key))
-                    continue;
-
-                if (found_key == key)
-                    markSecretArgument(i, /* argument_is_named= */ true);
-            }
-        }
-    };
-}
-
-
 void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const
 {
     /// These functions contain some unexpected ASTs in arguments (e.g. SETTINGS or even a SELECT query)
@@ -1195,7 +694,7 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
 
         FunctionSecretArgumentsFinder::Result secret_arguments;
         if (!settings.show_secrets)
-            secret_arguments = FunctionSecretArgumentsFinder{*this}.getResult();
+            secret_arguments = FunctionSecretArgumentsFinderAST(*this).getResult();
 
         for (size_t i = 0, size = arguments->children.size(); i < size; ++i)
         {
@@ -1260,7 +759,7 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
 
 bool ASTFunction::hasSecretParts() const
 {
-    return (FunctionSecretArgumentsFinder{*this}.getResult().hasSecrets()) || childrenHaveSecretParts();
+    return (FunctionSecretArgumentsFinderAST(*this).getResult().hasSecrets()) || childrenHaveSecretParts();
 }
 
 String getFunctionName(const IAST * ast)

src/Parsers/FunctionSecretArgumentsFinder.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <vector>
+
+namespace DB
+{
+
+class FunctionSecretArgumentsFinder
+{
+public:
+    struct Result
+    {
+        /// Result constructed by default means no arguments will be hidden.
+        size_t start = static_cast<size_t>(-1);
+        size_t count = 0; /// Mostly it's either 0 or 1. There are only a few cases where `count` can be greater than 1 (e.g. see `encrypt`).
+                            /// In all known cases secret arguments are consecutive
+        bool are_named = false; /// Arguments like `password = 'password'` are considered as named arguments.
+        /// E.g. "headers" in `url('..', headers('foo' = '[HIDDEN]'))`
+        std::vector<std::string> nested_maps;
+
+        bool hasSecrets() const
+        {
+            return count != 0 || !nested_maps.empty();
+        }
+    };
+};
+
+}

src/Parsers/FunctionSecretArgumentsFinderAST.h

@@ -0,0 +1,499 @@
+#pragma once
+
+#include <Parsers/FunctionSecretArgumentsFinder.h>
+#include <Core/QualifiedTableName.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Common/KnownObjectNames.h>
+
+#include <boost/algorithm/string/predicate.hpp>
+
+
+namespace DB
+{
+
+
+/// Finds arguments of a specified function which should not be displayed for most users for security reasons.
+/// That involves passwords and secret keys.
+class FunctionSecretArgumentsFinderAST
+{
+public:
+    explicit FunctionSecretArgumentsFinderAST(const ASTFunction & function_) : function(function_)
+    {
+        if (!function.arguments)
+            return;
+
+        const auto * expr_list = function.arguments->as<ASTExpressionList>();
+        if (!expr_list)
+            return;
+
+        arguments = &expr_list->children;
+        switch (function.kind)
+        {
+            case ASTFunction::Kind::ORDINARY_FUNCTION: findOrdinaryFunctionSecretArguments(); break;
+            case ASTFunction::Kind::WINDOW_FUNCTION: break;
+            case ASTFunction::Kind::LAMBDA_FUNCTION:  break;
+            case ASTFunction::Kind::TABLE_ENGINE: findTableEngineSecretArguments(); break;
+            case ASTFunction::Kind::DATABASE_ENGINE: findDatabaseEngineSecretArguments(); break;
+            case ASTFunction::Kind::BACKUP_NAME: findBackupNameSecretArguments(); break;
+        }
+    }
+
+    FunctionSecretArgumentsFinder::Result getResult() const { return result; }
+
+private:
+    const ASTFunction & function;
+    const ASTs * arguments = nullptr;
+    FunctionSecretArgumentsFinder::Result result;
+
+    void markSecretArgument(size_t index, bool argument_is_named = false)
+    {
+        if (index >= arguments->size())
+            return;
+        if (!result.count)
+        {
+            result.start = index;
+            result.are_named = argument_is_named;
+        }
+        chassert(index >= result.start); /// We always check arguments consecutively
+        result.count = index + 1 - result.start;
+        if (!argument_is_named)
+            result.are_named = false;
+    }
+
+    void findOrdinaryFunctionSecretArguments()
+    {
+        if ((function.name == "mysql") || (function.name == "postgresql") || (function.name == "mongodb"))
+        {
+            /// mysql('host:port', 'database', 'table', 'user', 'password', ...)
+            /// postgresql('host:port', 'database', 'table', 'user', 'password', ...)
+            /// mongodb('host:port', 'database', 'collection', 'user', 'password', ...)
+            findMySQLFunctionSecretArguments();
+        }
+        else if ((function.name == "s3") || (function.name == "cosn") || (function.name == "oss") ||
+                    (function.name == "deltaLake") || (function.name == "hudi") || (function.name == "iceberg"))
+        {
+            /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
+            findS3FunctionSecretArguments(/* is_cluster_function= */ false);
+        }
+        else if (function.name == "s3Cluster")
+        {
+            /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...)
+            findS3FunctionSecretArguments(/* is_cluster_function= */ true);
+        }
+        else if ((function.name == "remote") || (function.name == "remoteSecure"))
+        {
+            /// remote('addresses_expr', 'db', 'table', 'user', 'password', ...)
+            findRemoteFunctionSecretArguments();
+        }
+        else if ((function.name == "encrypt") || (function.name == "decrypt") ||
+                    (function.name == "aes_encrypt_mysql") || (function.name == "aes_decrypt_mysql") ||
+                    (function.name == "tryDecrypt"))
+        {
+            /// encrypt('mode', 'plaintext', 'key' [, iv, aad])
+            findEncryptionFunctionSecretArguments();
+        }
+        else if (function.name == "url")
+        {
+            findURLSecretArguments();
+        }
+    }
+
+    void findMySQLFunctionSecretArguments()
+    {
+        if (isNamedCollectionName(0))
+        {
+            /// mysql(named_collection, ..., password = 'password', ...)
+            findSecretNamedArgument("password", 1);
+        }
+        else
+        {
+            /// mysql('host:port', 'database', 'table', 'user', 'password', ...)
+            markSecretArgument(4);
+        }
+    }
+
+    /// Returns the number of arguments excluding "headers" and "extra_credentials" (which should
+    /// always be at the end). Marks "headers" as secret, if found.
+    size_t excludeS3OrURLNestedMaps()
+    {
+        size_t count = arguments->size();
+        while (count > 0)
+        {
+            const ASTFunction * f = arguments->at(count - 1)->as<ASTFunction>();
+            if (!f)
+                break;
+            if (f->name == "headers")
+                result.nested_maps.push_back(f->name);
+            else if (f->name != "extra_credentials")
+                break;
+            count -= 1;
+        }
+        return count;
+    }
+
+    void findS3FunctionSecretArguments(bool is_cluster_function)
+    {
+        /// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument.
+        size_t url_arg_idx = is_cluster_function ? 1 : 0;
+
+        if (!is_cluster_function && isNamedCollectionName(0))
+        {
+            /// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...)
+            findSecretNamedArgument("secret_access_key", 1);
+            return;
+        }
+
+        /// We should check other arguments first because we don't need to do any replacement in case of
+        /// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
+        /// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
+        size_t count = excludeS3OrURLNestedMaps();
+        if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4))
+        {
+            String second_arg;
+            if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg))
+            {
+                if (boost::iequals(second_arg, "NOSIGN"))
+                    return; /// The argument after 'url' is "NOSIGN".
+
+                if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg))
+                    return; /// The argument after 'url' is a format: s3('url', 'format', ...)
+            }
+        }
+
+        /// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures:
+        /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
+        /// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
+        if (url_arg_idx + 2 < count)
+            markSecretArgument(url_arg_idx + 2);
+    }
+
+    void findURLSecretArguments()
+    {
+        if (!isNamedCollectionName(0))
+            excludeS3OrURLNestedMaps();
+    }
+
+    bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const
+    {
+        if (arg_idx >= arguments->size())
+            return false;
+
+        return tryGetStringFromArgument(*(*arguments)[arg_idx], res, allow_identifier);
+    }
+
+    static bool tryGetStringFromArgument(const IAST & argument, String * res, bool allow_identifier = true)
+    {
+        if (const auto * literal = argument.as<ASTLiteral>())
+        {
+            if (literal->value.getType() != Field::Types::String)
+                return false;
+            if (res)
+                *res = literal->value.safeGet<String>();
+            return true;
+        }
+
+        if (allow_identifier)
+        {
+            if (const auto * id = argument.as<ASTIdentifier>())
+            {
+                if (res)
+                    *res = id->name();
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    void findRemoteFunctionSecretArguments()
+    {
+        if (isNamedCollectionName(0))
+        {
+            /// remote(named_collection, ..., password = 'password', ...)
+            findSecretNamedArgument("password", 1);
+            return;
+        }
+
+        /// We're going to replace 'password' with '[HIDDEN'] for the following signatures:
+        /// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key])
+        /// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key])
+        /// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key])
+
+        /// But we should check the number of arguments first because we don't need to do any replacements in case of
+        /// remote('addresses_expr', db.table)
+        if (arguments->size() < 3)
+            return;
+
+        size_t arg_num = 1;
+
+        /// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'.
+        const auto * table_function = (*arguments)[arg_num]->as<ASTFunction>();
+        if (table_function && KnownTableFunctionNames::instance().exists(table_function->name))
+        {
+            ++arg_num;
+        }
+        else
+        {
+            std::optional<String> database;
+            std::optional<QualifiedTableName> qualified_table_name;
+            if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name))
+            {
+                /// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'.
+                /// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user'
+                /// before the argument 'password'. So it's safer to wipe two arguments just in case.
+                /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
+                /// before wiping it (because the `password` argument is always a literal string).
+                if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false))
+                {
+                    /// Wipe either `password` or `user`.
+                    markSecretArgument(arg_num + 2);
+                }
+                if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false))
+                {
+                    /// Wipe either `password` or `sharding_key`.
+                    markSecretArgument(arg_num + 3);
+                }
+                return;
+            }
+
+            /// Skip the current argument (which is either a database name or a qualified table name).
+            ++arg_num;
+            if (database)
+            {
+                /// Skip the 'table' argument if the previous argument was a database name.
+                ++arg_num;
+            }
+        }
+
+        /// Skip username.
+        ++arg_num;
+
+        /// Do our replacement:
+        /// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...)
+        /// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
+        /// before wiping it (because the `password` argument is always a literal string).
+        bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false);
+        if (can_be_password)
+            markSecretArgument(arg_num);
+    }
+
+    /// Tries to get either a database name or a qualified table name from an argument.
+    /// Empty string is also allowed (it means the default database).
+    /// The function is used by findRemoteFunctionSecretArguments() to determine how many arguments to skip before a password.
+    bool tryGetDatabaseNameOrQualifiedTableName(
+        size_t arg_idx,
+        std::optional<String> & res_database,
+        std::optional<QualifiedTableName> & res_qualified_table_name) const
+    {
+        res_database.reset();
+        res_qualified_table_name.reset();
+
+        String str;
+        if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true))
+            return false;
+
+        if (str.empty())
+        {
+            res_database = "";
+            return true;
+        }
+
+        auto qualified_table_name = QualifiedTableName::tryParseFromString(str);
+        if (!qualified_table_name)
+            return false;
+
+        if (qualified_table_name->database.empty())
+            res_database = std::move(qualified_table_name->table);
+        else
+            res_qualified_table_name = std::move(qualified_table_name);
+        return true;
+    }
+
+    void findEncryptionFunctionSecretArguments()
+    {
+        if (arguments->empty())
+            return;
+
+        /// We replace all arguments after 'mode' with '[HIDDEN]':
+        /// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]')
+        result.start = 1;
+        result.count = arguments->size() - 1;
+    }
+
+    void findTableEngineSecretArguments()
+    {
+        const String & engine_name = function.name;
+        if (engine_name == "ExternalDistributed")
+        {
+            /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password')
+            findExternalDistributedTableEngineSecretArguments();
+        }
+        else if ((engine_name == "MySQL") || (engine_name == "PostgreSQL") ||
+                    (engine_name == "MaterializedPostgreSQL") || (engine_name == "MongoDB"))
+        {
+            /// MySQL('host:port', 'database', 'table', 'user', 'password', ...)
+            /// PostgreSQL('host:port', 'database', 'table', 'user', 'password', ...)
+            /// MaterializedPostgreSQL('host:port', 'database', 'table', 'user', 'password', ...)
+            /// MongoDB('host:port', 'database', 'collection', 'user', 'password', ...)
+            findMySQLFunctionSecretArguments();
+        }
+        else if ((engine_name == "S3") || (engine_name == "COSN") || (engine_name == "OSS") ||
+                    (engine_name == "DeltaLake") || (engine_name == "Hudi") || (engine_name == "Iceberg") || (engine_name == "S3Queue"))
+        {
+            /// S3('url', ['aws_access_key_id', 'aws_secret_access_key',] ...)
+            findS3TableEngineSecretArguments();
+        }
+        else if (engine_name == "URL")
+        {
+            findURLSecretArguments();
+        }
+    }
+
+    void findExternalDistributedTableEngineSecretArguments()
+    {
+        if (isNamedCollectionName(1))
+        {
+            /// ExternalDistributed('engine', named_collection, ..., password = 'password', ...)
+            findSecretNamedArgument("password", 2);
+        }
+        else
+        {
+            /// ExternalDistributed('engine', 'host:port', 'database', 'table', 'user', 'password')
+            markSecretArgument(5);
+        }
+    }
+
+    void findS3TableEngineSecretArguments()
+    {
+        if (isNamedCollectionName(0))
+        {
+            /// S3(named_collection, ..., secret_access_key = 'secret_access_key')
+            findSecretNamedArgument("secret_access_key", 1);
+            return;
+        }
+
+        /// We should check other arguments first because we don't need to do any replacement in case of
+        /// S3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
+        /// S3('url', 'format', 'compression' [, extra_credentials(..)] [, headers(..)])
+        size_t count = excludeS3OrURLNestedMaps();
+        if ((3 <= count) && (count <= 4))
+        {
+            String second_arg;
+            if (tryGetStringFromArgument(1, &second_arg))
+            {
+                if (boost::iequals(second_arg, "NOSIGN"))
+                    return; /// The argument after 'url' is "NOSIGN".
+
+                if (count == 3)
+                {
+                    if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg))
+                        return; /// The argument after 'url' is a format: S3('url', 'format', ...)
+                }
+            }
+        }
+
+        /// We replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures:
+        /// S3('url', 'aws_access_key_id', 'aws_secret_access_key')
+        /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format')
+        /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
+        if (2 < count)
+            markSecretArgument(2);
+    }
+
+    void findDatabaseEngineSecretArguments()
+    {
+        const String & engine_name = function.name;
+        if ((engine_name == "MySQL") || (engine_name == "MaterializeMySQL") ||
+            (engine_name == "MaterializedMySQL") || (engine_name == "PostgreSQL") ||
+            (engine_name == "MaterializedPostgreSQL"))
+        {
+            /// MySQL('host:port', 'database', 'user', 'password')
+            /// PostgreSQL('host:port', 'database', 'user', 'password')
+            findMySQLDatabaseSecretArguments();
+        }
+        else if (engine_name == "S3")
+        {
+            /// S3('url', 'access_key_id', 'secret_access_key')
+            findS3DatabaseSecretArguments();
+        }
+    }
+
+    void findMySQLDatabaseSecretArguments()
+    {
+        if (isNamedCollectionName(0))
+        {
+            /// MySQL(named_collection, ..., password = 'password', ...)
+            findSecretNamedArgument("password", 1);
+        }
+        else
+        {
+            /// MySQL('host:port', 'database', 'user', 'password')
+            markSecretArgument(3);
+        }
+    }
+
+    void findS3DatabaseSecretArguments()
+    {
+        if (isNamedCollectionName(0))
+        {
+            /// S3(named_collection, ..., secret_access_key = 'password', ...)
+            findSecretNamedArgument("secret_access_key", 1);
+        }
+        else
+        {
+            /// S3('url', 'access_key_id', 'secret_access_key')
+            markSecretArgument(2);
+        }
+    }
+
+    void findBackupNameSecretArguments()
+    {
+        const String & engine_name = function.name;
+        if (engine_name == "S3")
+        {
+            /// BACKUP ... TO S3(url, [aws_access_key_id, aws_secret_access_key])
+            markSecretArgument(2);
+        }
+    }
+
+    /// Whether a specified argument can be the name of a named collection?
+    bool isNamedCollectionName(size_t arg_idx) const
+    {
+        if (arguments->size() <= arg_idx)
+            return false;
+
+        const auto * identifier = (*arguments)[arg_idx]->as<ASTIdentifier>();
+        return identifier != nullptr;
+    }
+
+    /// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified.
+    void findSecretNamedArgument(const std::string_view & key, size_t start = 0)
+    {
+        for (size_t i = start; i < arguments->size(); ++i)
+        {
+            const auto & argument = (*arguments)[i];
+            const auto * equals_func = argument->as<ASTFunction>();
+            if (!equals_func || (equals_func->name != "equals"))
+                continue;
+
+            const auto * expr_list = equals_func->arguments->as<ASTExpressionList>();
+            if (!expr_list)
+                continue;
+
+            const auto & equal_args = expr_list->children;
+            if (equal_args.size() != 2)
+                continue;
+
+            String found_key;
+            if (!tryGetStringFromArgument(*equal_args[0], &found_key))
+                continue;
+
+            if (found_key == key)
+                markSecretArgument(i, /* argument_is_named= */ true);
+        }
+    }
+};
+
+}

src/Planner/Planner.cpp

@@ -1373,7 +1373,7 @@ void Planner::buildPlanForQueryNode()
     const auto & settings = query_context->getSettingsRef();
     if (query_context->canUseTaskBasedParallelReplicas())
     {
-        if (planner_context->getPreparedSets().hasSubqueries())
+        if (!settings.parallel_replicas_allow_in_with_subquery && planner_context->getPreparedSets().hasSubqueries())
         {
             if (settings.allow_experimental_parallel_reading_from_replicas >= 2)
                 throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "IN with subquery is not supported with parallel replicas");

src/Planner/findParallelReplicasQuery.cpp

@@ -2,6 +2,7 @@
 #include <Interpreters/ClusterProxy/SelectStreamFactory.h>
 #include <Interpreters/InterpreterSelectQueryAnalyzer.h>
 #include <Processors/QueryPlan/JoinStep.h>
+#include <Processors/QueryPlan/CreatingSetsStep.h>
 #include <Storages/buildQueryTreeForShard.h>
 #include <Interpreters/ClusterProxy/executeQuery.h>
 #include <Planner/PlannerJoinTree.h>
@@ -156,7 +157,8 @@ QueryTreeNodePtr replaceTablesWithDummyTables(const QueryTreeNodePtr & query, co
 /// Otherwise we can execute current query up to WithMergableStage only.
 const QueryNode * findQueryForParallelReplicas(
     std::stack<const QueryNode *> stack,
-    const std::unordered_map<const QueryNode *, const QueryPlan::Node *> & mapping)
+    const std::unordered_map<const QueryNode *, const QueryPlan::Node *> & mapping,
+    const Settings & settings)
 {
     const QueryPlan::Node * prev_checked_node = nullptr;
     const QueryNode * res = nullptr;
@@ -192,7 +194,11 @@ const QueryNode * findQueryForParallelReplicas(
             {
                 const auto * expression = typeid_cast<ExpressionStep *>(step);
                 const auto * filter = typeid_cast<FilterStep *>(step);
-                if (!expression && !filter)
+
+                const auto * creating_sets = typeid_cast<DelayedCreatingSetsStep *>(step);
+                bool allowed_creating_sets = settings.parallel_replicas_allow_in_with_subquery && creating_sets;
+
+                if (!expression && !filter && !allowed_creating_sets)
                     can_distribute_full_node = false;
 
                 next_node_to_check = children.front();
@@ -274,7 +280,7 @@ const QueryNode * findQueryForParallelReplicas(const QueryTreeNodePtr & query_tr
     /// So that we build a list of candidates again, and call findQueryForParallelReplicas for it.
     auto new_stack = getSupportingParallelReplicasQuery(updated_query_tree.get());
     const auto & mapping = planner.getQueryNodeToPlanStepMapping();
-    const auto * res = findQueryForParallelReplicas(new_stack, mapping);
+    const auto * res = findQueryForParallelReplicas(new_stack, mapping, context->getSettingsRef());
 
     /// Now, return a query from initial stack.
     if (res)

src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp

@@ -12,6 +12,7 @@
 #include <Columns/IColumn.h>
 
 #include <Common/assert_cast.h>
+#include "DataTypes/IDataType.h"
 
 #include <DataTypes/DataTypeMap.h>
 #include <DataTypes/DataTypeNullable.h>
@@ -35,9 +36,12 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
+namespace
+{
+
 constexpr auto FORMAT_NAME = "Prometheus";
 
-static bool isDataTypeMapString(const DataTypePtr & type)
+bool isDataTypeMapString(const DataTypePtr & type)
 {
     if (!isMap(type))
         return false;
@@ -45,8 +49,8 @@ static bool isDataTypeMapString(const DataTypePtr & type)
     return isStringOrFixedString(type_map->getKeyType()) && isStringOrFixedString(type_map->getValueType());
 }
 
-template <typename ResType, typename Pred>
-static void getColumnPos(const Block & header, const String & col_name, Pred pred, ResType & res)
+template <typename ResType>
+void getColumnPos(const Block & header, const String & col_name, bool (*pred)(const DataTypePtr &), ResType & res)
 {
     static_assert(std::is_same_v<ResType, size_t> || std::is_same_v<ResType, std::optional<size_t>>, "Illegal ResType");
 
@@ -71,7 +75,7 @@ static void getColumnPos(const Block & header, const String & col_name, Pred pre
     }
 }
 
-static Float64 tryParseFloat(const String & s)
+Float64 tryParseFloat(const String & s)
 {
     Float64 t = 0;
     ReadBufferFromString buf(s);
@@ -79,6 +83,8 @@ static Float64 tryParseFloat(const String & s)
     return t;
 }
 
+}
+
 PrometheusTextOutputFormat::PrometheusTextOutputFormat(
     WriteBuffer & out_,
     const Block & header_,
@@ -89,12 +95,12 @@ PrometheusTextOutputFormat::PrometheusTextOutputFormat(
 {
     const Block & header = getPort(PortKind::Main).getHeader();
 
-    getColumnPos(header, "name", isStringOrFixedString<DataTypePtr>, pos.name);
-    getColumnPos(header, "value", isNumber<DataTypePtr>, pos.value);
+    getColumnPos(header, "name", isStringOrFixedString, pos.name);
+    getColumnPos(header, "value", isNumber, pos.value);
 
-    getColumnPos(header, "help", isStringOrFixedString<DataTypePtr>, pos.help);
-    getColumnPos(header, "type", isStringOrFixedString<DataTypePtr>, pos.type);
-    getColumnPos(header, "timestamp", isNumber<DataTypePtr>, pos.timestamp);
+    getColumnPos(header, "help", isStringOrFixedString, pos.help);
+    getColumnPos(header, "type", isStringOrFixedString, pos.type);
+    getColumnPos(header, "timestamp", isNumber, pos.timestamp);
     getColumnPos(header, "labels", isDataTypeMapString, pos.labels);
 }
 

src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp

@@ -12,7 +12,7 @@
 #include <DataTypes/NestedUtils.h>
 #include <DataTypes/DataTypeLowCardinality.h>
 #include <IO/WriteHelpers.h>
-#include <Storages/BlockNumberColumn.h>
+#include <Storages/MergeTree/MergeTreeVirtualColumns.h>
 
 
 namespace DB

src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp

@@ -431,7 +431,7 @@ AggregateProjectionCandidates getAggregateProjectionCandidates(
 {
     const auto & keys = aggregating.getParams().keys;
     const auto & aggregates = aggregating.getParams().aggregates;
-    Block key_virtual_columns = reading.getMergeTreeData().getSampleBlockWithVirtualColumns();
+    Block key_virtual_columns = reading.getMergeTreeData().getHeaderWithVirtualsForFilter();
 
     AggregateProjectionCandidates candidates;
 

src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp

@@ -135,7 +135,7 @@ bool optimizeUseNormalProjections(Stack & stack, QueryPlan::Nodes & nodes)
     std::list<NormalProjectionCandidate> candidates;
     NormalProjectionCandidate * best_candidate = nullptr;
 
-    const Names & required_columns = reading->getRealColumnNames();
+    const Names & required_columns = reading->getAllColumnNames();
     const auto & parts = reading->getParts();
     const auto & alter_conversions = reading->getAlterConvertionsForParts();
     const auto & query_info = reading->getQueryInfo();

src/Processors/QueryPlan/ReadFromMergeTree.cpp

@@ -261,30 +261,24 @@ void ReadFromMergeTree::AnalysisResult::checkLimits(const Settings & settings, c
 ReadFromMergeTree::ReadFromMergeTree(
     MergeTreeData::DataPartsVector parts_,
     std::vector<AlterConversionsPtr> alter_conversions_,
-    const Names & column_names_,
-    Names real_column_names_,
-    Names virt_column_names_,
+    Names all_column_names_,
     const MergeTreeData & data_,
     const SelectQueryInfo & query_info_,
     const StorageSnapshotPtr & storage_snapshot_,
     const ContextPtr & context_,
     size_t max_block_size_,
     size_t num_streams_,
-    bool sample_factor_column_queried_,
     std::shared_ptr<PartitionIdToMaxBlock> max_block_numbers_to_read_,
     LoggerPtr log_,
     AnalysisResultPtr analyzed_result_ptr_,
     bool enable_parallel_reading)
     : SourceStepWithFilter(DataStream{.header = MergeTreeSelectProcessor::transformHeader(
-        storage_snapshot_->getSampleBlockForColumns(real_column_names_),
-        query_info_.prewhere_info,
-        data_.getPartitionValueType(),
-        virt_column_names_)}, column_names_, query_info_, storage_snapshot_, context_)
+        storage_snapshot_->getSampleBlockForColumns(all_column_names_),
+        query_info_.prewhere_info)}, all_column_names_, query_info_, storage_snapshot_, context_)
     , reader_settings(getMergeTreeReaderSettings(context_, query_info_))
     , prepared_parts(std::move(parts_))
     , alter_conversions_for_parts(std::move(alter_conversions_))
-    , real_column_names(std::move(real_column_names_))
-    , virt_column_names(std::move(virt_column_names_))
+    , all_column_names(std::move(all_column_names_))
     , data(data_)
     , actions_settings(ExpressionActionsSettings::fromContext(context_))
     , metadata_for_reading(storage_snapshot->getMetadataForQuery())
@@ -293,20 +287,11 @@ ReadFromMergeTree::ReadFromMergeTree(
         .preferred_block_size_bytes = context->getSettingsRef().preferred_block_size_bytes,
         .preferred_max_column_in_block_size_bytes = context->getSettingsRef().preferred_max_column_in_block_size_bytes}
     , requested_num_streams(num_streams_)
-    , sample_factor_column_queried(sample_factor_column_queried_)
     , max_block_numbers_to_read(std::move(max_block_numbers_to_read_))
     , log(std::move(log_))
     , analyzed_result_ptr(analyzed_result_ptr_)
     , is_parallel_reading_from_replicas(enable_parallel_reading)
 {
-    if (sample_factor_column_queried)
-    {
-        /// Only _sample_factor virtual column is added by ReadFromMergeTree
-        /// Other virtual columns are added by MergeTreeSelectProcessor.
-        auto type = std::make_shared<DataTypeFloat64>();
-        output_stream->header.insert({type->createColumn(), type, "_sample_factor"});
-    }
-
     if (is_parallel_reading_from_replicas)
     {
         all_ranges_callback = context->getMergeTreeAllRangesCallback();
@@ -368,12 +353,12 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas(
     auto pool = std::make_shared<MergeTreeReadPoolParallelReplicas>(
         std::move(extension),
         std::move(parts_with_range),
+        shared_virtual_fields,
         storage_snapshot,
         prewhere_info,
         actions_settings,
         reader_settings,
         required_columns,
-        virt_column_names,
         pool_settings,
         context);
 
@@ -387,8 +372,8 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas(
         auto algorithm = std::make_unique<MergeTreeThreadSelectAlgorithm>(i);
 
         auto processor = std::make_unique<MergeTreeSelectProcessor>(
-            pool, std::move(algorithm), data, prewhere_info,
-            actions_settings, block_size_copy, reader_settings, virt_column_names);
+            pool, std::move(algorithm), storage_snapshot, prewhere_info,
+            actions_settings, block_size_copy, reader_settings);
 
         auto source = std::make_shared<MergeTreeSource>(std::move(processor));
         pipes.emplace_back(std::move(source));
@@ -449,12 +434,12 @@ Pipe ReadFromMergeTree::readFromPool(
     {
         pool = std::make_shared<MergeTreePrefetchedReadPool>(
             std::move(parts_with_range),
+            shared_virtual_fields,
             storage_snapshot,
             prewhere_info,
             actions_settings,
             reader_settings,
             required_columns,
-            virt_column_names,
             pool_settings,
             context);
     }
@@ -462,12 +447,12 @@ Pipe ReadFromMergeTree::readFromPool(
     {
         pool = std::make_shared<MergeTreeReadPool>(
             std::move(parts_with_range),
+            shared_virtual_fields,
             storage_snapshot,
             prewhere_info,
             actions_settings,
             reader_settings,
             required_columns,
-            virt_column_names,
             pool_settings,
             context);
     }
@@ -486,8 +471,8 @@ Pipe ReadFromMergeTree::readFromPool(
         auto algorithm = std::make_unique<MergeTreeThreadSelectAlgorithm>(i);
 
         auto processor = std::make_unique<MergeTreeSelectProcessor>(
-            pool, std::move(algorithm), data, prewhere_info,
-            actions_settings, block_size_copy, reader_settings, virt_column_names);
+            pool, std::move(algorithm), storage_snapshot, prewhere_info,
+            actions_settings, block_size_copy, reader_settings);
 
         auto source = std::make_shared<MergeTreeSource>(std::move(processor));
 
@@ -538,12 +523,12 @@ Pipe ReadFromMergeTree::readInOrder(
             std::move(extension),
             mode,
             parts_with_ranges,
+            shared_virtual_fields,
             storage_snapshot,
             prewhere_info,
             actions_settings,
             reader_settings,
             required_columns,
-            virt_column_names,
             pool_settings,
             context);
     }
@@ -553,12 +538,12 @@ Pipe ReadFromMergeTree::readInOrder(
             has_limit_below_one_block,
             read_type,
             parts_with_ranges,
+            shared_virtual_fields,
             storage_snapshot,
             prewhere_info,
             actions_settings,
             reader_settings,
             required_columns,
-            virt_column_names,
             pool_settings,
             context);
     }
@@ -592,8 +577,8 @@ Pipe ReadFromMergeTree::readInOrder(
             algorithm = std::make_unique<MergeTreeInOrderSelectAlgorithm>(i);
 
         auto processor = std::make_unique<MergeTreeSelectProcessor>(
-            pool, std::move(algorithm), data, prewhere_info,
-            actions_settings, block_size, reader_settings, virt_column_names);
+            pool, std::move(algorithm), storage_snapshot, prewhere_info,
+            actions_settings, block_size, reader_settings);
 
         processor->addPartLevelToChunk(isQueryWithFinal());
 
@@ -1302,8 +1287,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
         requested_num_streams,
         max_block_numbers_to_read,
         data,
-        real_column_names,
-        sample_factor_column_queried,
+        all_column_names,
         log,
         indexes);
 }
@@ -1489,8 +1473,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
     size_t num_streams,
     std::shared_ptr<PartitionIdToMaxBlock> max_block_numbers_to_read,
     const MergeTreeData & data,
-    const Names & real_column_names,
-    bool sample_factor_column_queried,
+    const Names & all_column_names,
     LoggerPtr log,
     std::optional<Indexes> & indexes)
 {
@@ -1503,8 +1486,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
         num_streams,
         max_block_numbers_to_read,
         data,
-        real_column_names,
-        sample_factor_column_queried,
+        all_column_names,
         log,
         indexes);
 }
@@ -1518,8 +1500,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToReadImpl(
     size_t num_streams,
     std::shared_ptr<PartitionIdToMaxBlock> max_block_numbers_to_read,
     const MergeTreeData & data,
-    const Names & real_column_names,
-    bool sample_factor_column_queried,
+    const Names & all_column_names,
     LoggerPtr log,
     std::optional<Indexes> & indexes)
 {
@@ -1528,7 +1509,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToReadImpl(
 
     size_t total_parts = parts.size();
 
-    result.column_names_to_read = real_column_names;
+    result.column_names_to_read = all_column_names;
 
     /// If there are only virtual columns in the query, you must request at least one non-virtual one.
     if (result.column_names_to_read.empty())
@@ -1587,7 +1568,6 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToReadImpl(
             data,
             metadata_snapshot,
             context_,
-            sample_factor_column_queried,
             log);
 
         if (result.sampling.read_nothing)
@@ -1704,10 +1684,8 @@ void ReadFromMergeTree::updatePrewhereInfo(const PrewhereInfoPtr & prewhere_info
     prewhere_info = prewhere_info_value;
 
     output_stream = DataStream{.header = MergeTreeSelectProcessor::transformHeader(
-        storage_snapshot->getSampleBlockForColumns(real_column_names),
-        prewhere_info_value,
-        data.getPartitionValueType(),
-        virt_column_names)};
+        storage_snapshot->getSampleBlockForColumns(all_column_names),
+        prewhere_info_value)};
 
     updateSortDescriptionForOutputStream(
         *output_stream,
@@ -1901,6 +1879,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
     storage_snapshot->data = std::make_unique<MergeTreeData::SnapshotData>();
 
     result.checkLimits(context->getSettingsRef(), query_info);
+    shared_virtual_fields.emplace("_sample_factor", result.sampling.used_sample_factor);
 
     LOG_DEBUG(
         log,
@@ -1985,18 +1964,6 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
             result_projection = ActionsDAG::merge(std::move(*result_projection), std::move(*actions));
     };
 
-    /// By the way, if a distributed query or query to a Merge table is made, then the `_sample_factor` column can have different values.
-    if (sample_factor_column_queried)
-    {
-        ColumnWithTypeAndName column;
-        column.name = "_sample_factor";
-        column.type = std::make_shared<DataTypeFloat64>();
-        column.column = column.type->createColumnConst(0, Field(result.sampling.used_sample_factor));
-
-        auto adding_column = ActionsDAG::makeAddingColumnActions(std::move(column));
-        append_actions(std::move(adding_column));
-    }
-
     if (result_projection)
         cur_header = result_projection->updateHeader(cur_header);
 

src/Processors/QueryPlan/ReadFromMergeTree.h

@@ -110,16 +110,13 @@ public:
     ReadFromMergeTree(
         MergeTreeData::DataPartsVector parts_,
         std::vector<AlterConversionsPtr> alter_conversions_,
-        const Names & column_names_,
-        Names real_column_names_,
-        Names virt_column_names_,
+        Names all_column_names_,
         const MergeTreeData & data_,
         const SelectQueryInfo & query_info_,
         const StorageSnapshotPtr & storage_snapshot,
         const ContextPtr & context_,
         size_t max_block_size_,
         size_t num_streams_,
-        bool sample_factor_column_queried_,
         std::shared_ptr<PartitionIdToMaxBlock> max_block_numbers_to_read_,
         LoggerPtr log_,
         AnalysisResultPtr analyzed_result_ptr_,
@@ -136,8 +133,7 @@ public:
     void describeActions(JSONBuilder::JSONMap & map) const override;
     void describeIndexes(JSONBuilder::JSONMap & map) const override;
 
-    const Names & getRealColumnNames() const { return real_column_names; }
-    const Names & getVirtualColumnNames() const { return virt_column_names; }
+    const Names & getAllColumnNames() const { return all_column_names; }
 
     StorageID getStorageID() const { return data.getStorageID(); }
     UInt64 getSelectedParts() const { return selected_parts; }
@@ -164,8 +160,7 @@ public:
         size_t num_streams,
         std::shared_ptr<PartitionIdToMaxBlock> max_block_numbers_to_read,
         const MergeTreeData & data,
-        const Names & real_column_names,
-        bool sample_factor_column_queried,
+        const Names & all_column_names,
         LoggerPtr log,
         std::optional<Indexes> & indexes);
 
@@ -209,8 +204,7 @@ private:
         size_t num_streams,
         std::shared_ptr<PartitionIdToMaxBlock> max_block_numbers_to_read,
         const MergeTreeData & data,
-        const Names & real_column_names,
-        bool sample_factor_column_queried,
+        const Names & all_column_names,
         LoggerPtr log,
         std::optional<Indexes> & indexes);
 
@@ -227,8 +221,7 @@ private:
     MergeTreeData::DataPartsVector prepared_parts;
     std::vector<AlterConversionsPtr> alter_conversions_for_parts;
 
-    Names real_column_names;
-    Names virt_column_names;
+    Names all_column_names;
 
     const MergeTreeData & data;
     ExpressionActionsSettings actions_settings;
@@ -239,7 +232,6 @@ private:
 
     size_t requested_num_streams;
     size_t output_streams_limit = 0;
-    const bool sample_factor_column_queried;
 
     /// Used for aggregation optimization (see DB::QueryPlanOptimizations::tryAggregateEachPartitionIndependently).
     bool output_each_partition_through_separate_port = false;
@@ -280,7 +272,9 @@ private:
         RangesInDataParts && parts, size_t num_streams, const Names & origin_column_names, const Names & column_names, ActionsDAGPtr & out_projection);
 
     ReadFromMergeTree::AnalysisResult getAnalysisResult() const;
+
     AnalysisResultPtr analyzed_result_ptr;
+    VirtualFields shared_virtual_fields;
 
     bool is_parallel_reading_from_replicas;
     std::optional<MergeTreeAllRangesCallback> all_ranges_callback;