Merge branch 'master' into feature/dotnet-integration-test

base/common/DateLUTImpl.h

@@ -18,6 +18,8 @@
 
 #define DATE_LUT_MAX (0xFFFFFFFFU - 86400)
 #define DATE_LUT_MAX_DAY_NUM 0xFFFF
+/// Max int value of Date32, DATE LUT cache size minus daynum_offset_epoch
+#define DATE_LUT_MAX_EXTEND_DAY_NUM (DATE_LUT_SIZE - 16436)
 
 /// A constant to add to time_t so every supported time point becomes non-negative and still has the same remainder of division by 3600.
 /// If we treat "remainder of division" operation in the sense of modular arithmetic (not like in C++).
@@ -270,6 +272,8 @@ public:
     auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; }
     auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; }
 
+    auto getDayNumOffsetEpoch() const { return daynum_offset_epoch; }
+
     /// All functions below are thread-safe; arguments are not checked.
 
     inline ExtendedDayNum toDayNum(ExtendedDayNum d) const
@@ -926,15 +930,17 @@ public:
     {
         if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31))
             return LUTIndex(0);
-
+        auto year_lut_index = (year - DATE_LUT_MIN_YEAR) * 12 + month - 1;
-        return LUTIndex{years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1};
+        UInt32 index = years_months_lut[year_lut_index].toUnderType() + day_of_month - 1;
+        /// When date is out of range, default value is DATE_LUT_SIZE - 1 (2283-11-11)
+        return LUTIndex{std::min(index, static_cast<UInt32>(DATE_LUT_SIZE - 1))};
     }
 
     /// Create DayNum from year, month, day of month.
-    inline ExtendedDayNum makeDayNum(Int16 year, UInt8 month, UInt8 day_of_month) const
+    inline ExtendedDayNum makeDayNum(Int16 year, UInt8 month, UInt8 day_of_month, Int32 default_error_day_num = 0) const
     {
         if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31))
-            return ExtendedDayNum(0);
+            return ExtendedDayNum(default_error_day_num);
 
         return toDayNum(makeLUTIndex(year, month, day_of_month));
     }
@@ -1091,9 +1097,9 @@ public:
         return lut[new_index].date + time;
     }
 
-    inline NO_SANITIZE_UNDEFINED Time addWeeks(Time t, Int64 delta) const
+    inline NO_SANITIZE_UNDEFINED Time addWeeks(Time t, Int32 delta) const
     {
-        return addDays(t, delta * 7);
+        return addDays(t, static_cast<Int64>(delta) * 7);
     }
 
     inline UInt8 saturateDayOfMonth(Int16 year, UInt8 month, UInt8 day_of_month) const
@@ -1158,14 +1164,14 @@ public:
         return toDayNum(addMonthsIndex(d, delta));
     }
 
-    inline Time NO_SANITIZE_UNDEFINED addQuarters(Time t, Int64 delta) const
+    inline Time NO_SANITIZE_UNDEFINED addQuarters(Time t, Int32 delta) const
     {
-        return addMonths(t, delta * 3);
+        return addMonths(t, static_cast<Int64>(delta) * 3);
     }
 
-    inline ExtendedDayNum addQuarters(ExtendedDayNum d, Int64 delta) const
+    inline ExtendedDayNum addQuarters(ExtendedDayNum d, Int32 delta) const
     {
-        return addMonths(d, delta * 3);
+        return addMonths(d, static_cast<Int64>(delta) * 3);
     }
 
     template <typename DateOrTime>

base/common/LocalDate.h

@@ -70,6 +70,14 @@ public:
         m_day   = values.day_of_month;
     }
 
+    explicit LocalDate(ExtendedDayNum day_num)
+    {
+        const auto & values = DateLUT::instance().getValues(day_num);
+        m_year  = values.year;
+        m_month = values.month;
+        m_day   = values.day_of_month;
+    }
+
     LocalDate(unsigned short year_, unsigned char month_, unsigned char day_)
         : m_year(year_), m_month(month_), m_day(day_)
     {
@@ -98,6 +106,12 @@ public:
         return DayNum(lut.makeDayNum(m_year, m_month, m_day).toUnderType());
     }
 
+    ExtendedDayNum  getExtenedDayNum() const
+    {
+        const auto & lut = DateLUT::instance();
+        return ExtendedDayNum (lut.makeDayNum(m_year, m_month, m_day).toUnderType());
+    }
+
     operator DayNum() const
     {
         return getDayNum();

cmake/autogenerated_versions.txt

@@ -2,11 +2,11 @@
 
 # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54453)
+SET(VERSION_REVISION 54454)
 SET(VERSION_MAJOR 21)
-SET(VERSION_MINOR 8)
+SET(VERSION_MINOR 9)
 SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH fb895056568e26200629c7d19626e92d2dedc70d)
+SET(VERSION_GITHASH f48c5af90c2ad51955d1ee3b6b05d006b03e4238)
-SET(VERSION_DESCRIBE v21.8.1.1-prestable)
+SET(VERSION_DESCRIBE v21.9.1.1-prestable)
-SET(VERSION_STRING 21.8.1.1)
+SET(VERSION_STRING 21.9.1.1)
 # end of autochange

contrib/h3

@@ -1 +1 @@
-Subproject commit e209086ae1b5477307f545a0f6111780edc59940
+Subproject commit c7f46cfd71fb60e2fefc90e28abe81657deff735

contrib/h3-cmake/CMakeLists.txt

@@ -3,21 +3,22 @@ set(H3_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/h3/src/h3lib")
 
 set(SRCS
 "${H3_SOURCE_DIR}/lib/algos.c"
-"${H3_SOURCE_DIR}/lib/baseCells.c"
-"${H3_SOURCE_DIR}/lib/bbox.c"
 "${H3_SOURCE_DIR}/lib/coordijk.c"
-"${H3_SOURCE_DIR}/lib/faceijk.c"
+"${H3_SOURCE_DIR}/lib/bbox.c"
-"${H3_SOURCE_DIR}/lib/geoCoord.c"
-"${H3_SOURCE_DIR}/lib/h3Index.c"
-"${H3_SOURCE_DIR}/lib/h3UniEdge.c"
-"${H3_SOURCE_DIR}/lib/linkedGeo.c"
-"${H3_SOURCE_DIR}/lib/localij.c"
-"${H3_SOURCE_DIR}/lib/mathExtensions.c"
 "${H3_SOURCE_DIR}/lib/polygon.c"
+"${H3_SOURCE_DIR}/lib/h3Index.c"
 "${H3_SOURCE_DIR}/lib/vec2d.c"
 "${H3_SOURCE_DIR}/lib/vec3d.c"
 "${H3_SOURCE_DIR}/lib/vertex.c"
+"${H3_SOURCE_DIR}/lib/linkedGeo.c"
+"${H3_SOURCE_DIR}/lib/localij.c"
+"${H3_SOURCE_DIR}/lib/latLng.c"
+"${H3_SOURCE_DIR}/lib/directedEdge.c"
+"${H3_SOURCE_DIR}/lib/mathExtensions.c"
+"${H3_SOURCE_DIR}/lib/iterators.c"
 "${H3_SOURCE_DIR}/lib/vertexGraph.c"
+"${H3_SOURCE_DIR}/lib/faceijk.c"
+"${H3_SOURCE_DIR}/lib/baseCells.c"
 )
 
 configure_file("${H3_SOURCE_DIR}/include/h3api.h.in" "${H3_BINARY_DIR}/include/h3api.h")

debian/changelog

@@ -1,5 +1,5 @@
-clickhouse (21.8.1.1) unstable; urgency=low
+clickhouse (21.9.1.1) unstable; urgency=low
 
   * Modified source code
 
- -- clickhouse-release <clickhouse-release@yandex-team.ru>  Mon, 28 Jun 2021 00:50:15 +0300
+ -- clickhouse-release <clickhouse-release@yandex-team.ru>  Sat, 10 Jul 2021 08:22:49 +0300

docker/client/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:18.04
 
 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.8.1.*
+ARG version=21.9.1.*
 
 RUN apt-get update \
     && apt-get install --yes --no-install-recommends \

docker/server/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 
 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.8.1.*
+ARG version=21.9.1.*
 ARG gosu_ver=1.10
 
 # set non-empty deb_location_url url to create a docker image

docker/test/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:18.04
 
 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.8.1.*
+ARG version=21.9.1.*
 
 RUN apt-get update && \
     apt-get install -y apt-transport-https dirmngr && \

docs/en/development/adding_test_queries.md

@@ -105,11 +105,11 @@ clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee te
 
 5) ensure everything is correct, if the test output is incorrect (due to some bug for example), adjust the reference file using text editor.
 
-####  How to create good test
+#### How to create a good test
 
-- test should be
+- A test should be
 	- minimal - create only tables related to tested functionality, remove unrelated columns and parts of query
-	- fast - should not take longer than few seconds (better subseconds)
+	- fast - should not take longer than a few seconds (better subseconds)
 	- correct - fails then feature is not working
         - deterministic
 	- isolated / stateless 
@@ -126,6 +126,16 @@ clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee te
 - use other SQL files in the `0_stateless` folder as an example
 - ensure the feature / feature combination you want to test is not yet covered with existing tests
 
+#### Test naming rules
+
+It's important to name tests correctly, so one could turn some tests subset off in clickhouse-test invocation.
+
+| Tester flag| What should be in test name | When flag should be added |
+|---|---|---|---|
+| `--[no-]zookeeper`| "zookeeper" or "replica" | Test uses tables from ReplicatedMergeTree family |
+| `--[no-]shard` | "shard" or "distributed" or "global"| Test using connections to 127.0.0.2 or similar |
+| `--[no-]long` | "long" or "deadlock" or "race" | Test runs longer than 60 seconds |
+
 #### Commit / push / create PR.
 
 1) commit & push your changes 

docs/en/development/build.md

@@ -134,10 +134,10 @@ $ ./release
 
 ## Faster builds for development
 
-Normally all tools of the ClickHouse bundle, such as `clickhouse-server`, `clickhouse-client` etc., are linked into a single static executable, `clickhouse`. This executable must be re-linked on every change, which might be slow. Two common ways to improve linking time are to use `lld` linker, and use the 'split' build configuration, which builds a separate binary for every tool, and further splits the code into several shared libraries. To enable these tweaks, pass the following flags to `cmake`:
+Normally all tools of the ClickHouse bundle, such as `clickhouse-server`, `clickhouse-client` etc., are linked into a single static executable, `clickhouse`. This executable must be re-linked on every change, which might be slow. One common way to improve build time is to use the 'split' build configuration, which builds a separate binary for every tool, and further splits the code into several shared libraries. To enable this tweak, pass the following flags to `cmake`:
 
 ```
--DCMAKE_C_FLAGS="--ld-path=lld" -DCMAKE_CXX_FLAGS="--ld-path=lld" -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1
+-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1
 ```
 
 ## You Don’t Have to Build ClickHouse {#you-dont-have-to-build-clickhouse}

docs/en/operations/clickhouse-keeper.md

@@ -0,0 +1,114 @@
+---
+toc_priority: 66
+toc_title: ClickHouse Keeper
+---
+
+# [pre-production] clickhouse-keeper
+
+ClickHouse server use [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper.
+
+!!! warning "Warning"
+    This feature currently in pre-production stage. We test it in our CI and on small internal installations.
+
+## Implemetation details
+
+ZooKeeper is one of the first well-known open-source coordination systems. It's implemented in Java, has quite a simple and powerful data model. ZooKeeper's coordination algorithm called ZAB (ZooKeeper Atomic Broadcast) doesn't provide linearizability guarantees for reads, because each ZooKeeper node serves reads locally. Unlike ZooKeeper `clickhouse-keeper` written in C++ and use [RAFT algorithm](https://raft.github.io/) [implementation](https://github.com/eBay/NuRaft). This algorithm allows to have linearizability for reads and writes, has several open-source implementations in different languages.
+
+By default, `clickhouse-keeper` provides the same guarantees as ZooKeeper (linearizable writes, non-linearizable reads). It has a compatible client-server protocol, so any standard ZooKeeper client can be used to interact with `clickhouse-keeper`. Snapshots and logs have an incompatible format with ZooKeeper, but `clickhouse-keeper-converter` tool allows to convert ZooKeeper data to `clickhouse-keeper` snapshot. Interserver protocol in `clickhouse-keeper` also incompatible with ZooKeeper so mixed ZooKeeper/clickhouse-keeper cluster is impossible.
+
+## Configuration
+
+`clickhouse-keeper` can be used as a standalone replacement for ZooKeeper or as an internal part of the `clickhouse-server`, but in both cases configuration is almost the same `.xml` file. The main `clickhouse-keeper` configuration tag is `<keeper_server>`. Keeper configuration has the following parameters:
+
+-    `tcp_port` — the port for a client to connect (default for ZooKeeper is `2181`)
+-    `tcp_port_secure` — the secure port for a client to connect
+-    `server_id` — unique server id, each participant of the clickhouse-keeper cluster must have a unique number (1, 2, 3, and so on)
+-    `log_storage_path` — path to coordination logs, better to store logs on the non-busy device (same for ZooKeeper)
+-    `snapshot_storage_path` — path to coordination snapshots
+
+Other common parameters are inherited from clickhouse-server config (`listen_host`, `logger` and so on).
+
+Internal coordination settings are located in `<keeper_server>.<coordination_settings>` section:
+
+-    `operation_timeout_ms` — timeout for a single client operation
+-    `session_timeout_ms` — timeout for client session
+-    `dead_session_check_period_ms` — how often clickhouse-keeper check dead sessions and remove them
+-    `heart_beat_interval_ms` — how often a clickhouse-keeper leader will send heartbeats to followers
+-    `election_timeout_lower_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it can initiate leader election
+-    `election_timeout_upper_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it must initiate leader election
+-    `rotate_log_storage_interval` — how many logs to store in a single file
+-    `reserved_log_items` — how many coordination logs to store before compaction
+-    `snapshot_distance` — how often clickhouse-keeper will create new snapshots (in the number of logs)
+-    `snapshots_to_keep` — how many snapshots to keep
+-    `stale_log_gap` — the threshold when leader consider follower as stale and send snapshot to it instead of logs
+-    `force_sync` — call `fsync` on each write to coordination log
+-    `raft_logs_level` — text logging level about coordination (trace, debug, and so on)
+-    `shutdown_timeout` — wait to finish internal connections and shutdown
+-    `startup_timeout` — if the server doesn't connect to other quorum participants in the specified timeout it will terminate
+
+Quorum configuration is located in `<keeper_server>.<raft_configuration>` section and contain servers description. The only parameter for the whole quorum is `secure`, which enables encrypted connection for communication between quorum participants. The main parameters for each `<server>` are:
+
+-    `id` — server_id in quorum
+-    `hostname` — hostname where this server placed
+-    `port` — port where this server listen for connections
+
+
+Examples of configuration for quorum with three nodes can be found in [integration tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/integration) with `test_keeper_` prefix. Example configuration for server #1:
+
+```xml
+<keeper_server>
+    <tcp_port>2181</tcp_port>
+    <server_id>1</server_id>
+    <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+    <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+    <coordination_settings>
+        <operation_timeout_ms>10000</operation_timeout_ms>
+        <session_timeout_ms>30000</session_timeout_ms>
+        <raft_logs_level>trace</raft_logs_level>
+    </coordination_settings>
+
+    <raft_configuration>
+        <server>
+            <id>1</id>
+            <hostname>zoo1</hostname>
+            <port>9444</port>
+        </server>
+        <server>
+            <id>2</id>
+            <hostname>zoo2</hostname>
+            <port>9444</port>
+        </server>
+        <server>
+            <id>3</id>
+            <hostname>zoo3</hostname>
+            <port>9444</port>
+        </server>
+    </raft_configuration>
+</keeper_server>
+```
+
+## How to run
+
+`clickhouse-keeper` is bundled into `clickhouse-server` package, just add configuration of `<keeper_server>` and start clickhouse-server as always. If you want to run standalone `clickhouse-keeper` you can start it in a similar way with:
+
+```bash
+clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon
+```
+
+## [experimental] Migration from ZooKeeper
+
+Seamlessly migration from ZooKeeper to `clickhouse-keeper` is impossible you have to stop your ZooKeeper cluster, convert data and start `clickhouse-keeper`. `clickhouse-keeper-converter` tool allows to convert ZooKeeper logs and snapshots to `clickhouse-keeper` snapshot. It works only with ZooKeeper > 3.4. Steps for migration:
+
+1. Stop all ZooKeeper nodes.
+
+2. [optional, but recommended] Found ZooKeeper leader node, start and stop it again. It will force ZooKeeper to create consistent snapshot.
+
+3. Run `clickhouse-keeper-converter` on leader, example
+
+```bash
+clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/version-2 --output-dir /path/to/clickhouse/keeper/snapshots
+```
+
+4. Copy snapshot to `clickhouse-server` nodes with configured `keeper` or start `clickhouse-keeper` instead of ZooKeeper. Snapshot must persist only on leader node, leader will sync it automatically to other nodes.
+

docs/en/operations/configuration-files.md

@@ -22,6 +22,23 @@ Some settings specified in the main configuration file can be overridden in othe
 
 The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](../operations/server-configuration-parameters/settings.md)).
 
+If you want to replace an entire element with a substitution use `include` as element name.
+
+XML substitution example:
+
+```xml
+<yandex>
+    <!-- Appends XML subtree found at `/profiles-in-zookeeper` ZK path to `<profiles>` element. -->
+    <profiles from_zk="/profiles-in-zookeeper" />
+
+    <users>
+        <!-- Replaces `include` element with the subtree found at `/users-in-zookeeper` ZK path. -->
+        <include from_zk="/users-in-zookeeper" />
+        <include from_zk="/other-users-in-zookeeper" />
+    </users>
+</yandex>
+```
+
 Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at `/path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element.
 
 ## User Settings {#user-settings}
@@ -32,6 +49,8 @@ Users configuration can be splitted into separate files similar to `config.xml`
 Directory name is defined as `users_config` setting without `.xml` postfix concatenated with `.d`.
 Directory `users.d` is used by default, as `users_config` defaults to `users.xml`.
 
+Note that configuration files are first merged taking into account [Override](#override) settings and includes are processed after that.
+
 ## XML example {#example}
 
 For example, you can have separate config file for each user like this:

docs/en/sql-reference/functions/ext-dict-functions.md

@@ -12,7 +12,7 @@ For information on connecting and configuring external dictionaries, see [Extern
 
 ## dictGet, dictGetOrDefault, dictGetOrNull {#dictget}
 
-Retrieves values from an external dictionary. 
+Retrieves values from an external dictionary.
 
 ``` sql
 dictGet('dict_name', attr_names, id_expr)
@@ -24,7 +24,7 @@ dictGetOrNull('dict_name', attr_name, id_expr)
 
 -   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
 -   `attr_names` — Name of the column of the dictionary, [String literal](../../sql-reference/syntax.md#syntax-string-literal), or tuple of column names, [Tuple](../../sql-reference/data-types/tuple.md)([String literal](../../sql-reference/syntax.md#syntax-string-literal)).
--   `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
+-   `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning dictionary key-type value or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
 -   `default_value_expr` — Values returned if the dictionary does not contain a row with the `id_expr` key. [Expression](../../sql-reference/syntax.md#syntax-expressions) or [Tuple](../../sql-reference/data-types/tuple.md)([Expression](../../sql-reference/syntax.md#syntax-expressions)), returning the value (or values) in the data types configured for the `attr_names` attribute.
 
 **Returned value**
@@ -138,7 +138,7 @@ Configure the external dictionary:
                 <name>c2</name>
                 <type>String</type>
                 <null_value></null_value>
-            </attribute>            
+            </attribute>
         </structure>
         <lifetime>0</lifetime>
     </dictionary>
@@ -237,7 +237,7 @@ dictHas('dict_name', id_expr)
 **Arguments**
 
 -   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
--   `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
+-   `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning dictionary key-type value or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
 
 **Returned value**
 
@@ -292,16 +292,16 @@ Type: `UInt8`.
 
 Returns first-level children as an array of indexes. It is the inverse transformation for [dictGetHierarchy](#dictgethierarchy).
 
-**Syntax** 
+**Syntax**
 
 ``` sql
 dictGetChildren(dict_name, key)
 ```
 
-**Arguments** 
+**Arguments**
 
--   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). 
+-   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
--   `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value. 
+-   `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value.
 
 **Returned values**
 
@@ -339,7 +339,7 @@ SELECT dictGetChildren('hierarchy_flat_dictionary', number) FROM system.numbers
 
 ## dictGetDescendant {#dictgetdescendant}
 
-Returns all descendants as if [dictGetChildren](#dictgetchildren) function was applied `level` times recursively.  
+Returns all descendants as if [dictGetChildren](#dictgetchildren) function was applied `level` times recursively.
 
 **Syntax**
 
@@ -347,9 +347,9 @@ Returns all descendants as if [dictGetChildren](#dictgetchildren) function was a
 dictGetDescendants(dict_name, key, level)
 ```
 
-**Arguments** 
+**Arguments**
 
--   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). 
+-   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
 -   `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value.
 -   `level` — Hierarchy level. If `level = 0` returns all descendants to the end. [UInt8](../../sql-reference/data-types/int-uint.md).
 

docs/en/sql-reference/functions/tuple-functions.md

@@ -87,6 +87,8 @@ Result:
 └───────┴───────┘
 ```
 
+Note: the names are implementation specific and are subject to change. You should not assume specific names of the columns after application of the `untuple`.
+
 Example of using an `EXCEPT` expression:
 
 Query:

docs/zh/sql-reference/functions/bitmap-functions.md

@@ -81,7 +81,7 @@ SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,
 **示例**
 
 ``` sql
-SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res
+SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res
 ```
 
     ┌─res───────────────────────┐
@@ -174,7 +174,7 @@ SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS re
     │ [3] │
     └─────┘
 
-## 位图 {#bitmapor}
+## 位图或 {#bitmapor}
 
 为两个位图对象进行或操作，返回一个新的位图对象。
 

docs/zh/sql-reference/table-functions/mysql.md

@@ -1,13 +1,8 @@
----
-machine_translated: true
-machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
-toc_priority: 42
-toc_title: mysql
----
-
 # mysql {#mysql}
 
-允许 `SELECT` 要对存储在远程MySQL服务器上的数据执行的查询。
+允许对存储在远程MySQL服务器上的数据执行`SELECT`和`INSERT`查询。
+
+**语法**
 
 ``` sql
 mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']);
@@ -15,31 +10,44 @@ mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_
 
 **参数**
 
--   `host:port` — MySQL server address.
+-   `host:port` — MySQL服务器地址.
 
--   `database` — Remote database name.
+-   `database` — 远程数据库名称.
 
--   `table` — Remote table name.
+-   `table` — 远程表名称.
 
--   `user` — MySQL user.
+-   `user` — MySQL用户.
 
--   `password` — User password.
+-   `password` — 用户密码.
 
--   `replace_query` — Flag that converts `INSERT INTO` 查询到 `REPLACE INTO`. 如果 `replace_query=1`，查询被替换。
+-   `replace_query` — 将INSERT INTO` 查询转换为 `REPLACE INTO`的标志。如果 `replace_query=1`，查询被替换。
 
--   `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` 表达式被添加到 `INSERT` 查询。
+- `on_duplicate_clause` — 添加 `ON DUPLICATE KEY on_duplicate_clause` 表达式到 `INSERT` 查询。明确规定只能使用 `replace_query = 0` ，如果你同时设置replace_query = 1`和`on_duplicate_clause`，ClickHouse将产生异常。
 
-        Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See the MySQL documentation to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause.
+  示例：`INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`
 
-        To specify `on_duplicate_clause` you need to pass `0` to the `replace_query` parameter. If you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception.
+  `on_duplicate_clause`这里是`UPDATE c2 = c2 + 1`。请查阅MySQL文档，来找到可以和`ON DUPLICATE KEY`一起使用的 `on_duplicate_clause`子句。
 
-简单 `WHERE` 条款如 `=, !=, >, >=, <, <=` 当前在MySQL服务器上执行。
+简单的 `WHERE` 子句如 `=, !=, >, >=, <, <=` 将即时在MySQL服务器上执行。其余的条件和 `LIMIT` 只有在对MySQL的查询完成后，才会在ClickHouse中执行采样约束。
 
-其余的条件和 `LIMIT` 只有在对MySQL的查询完成后，才会在ClickHouse中执行采样约束。
+支持使用`|`并列进行多副本查询，示例如下：
+
+```sql
+SELECT name FROM mysql(`mysql{1|2|3}:3306`, 'mysql_database', 'mysql_table', 'user', 'password');
+```
+
+或
+
+```sql
+SELECT name FROM mysql(`mysql1:3306|mysql2:3306|mysql3:3306`, 'mysql_database', 'mysql_table', 'user', 'password');
+```
 
 **返回值**
 
-与原始MySQL表具有相同列的table对象。
+与原始MySQL表具有相同列的表对象。
+
+!!! note "注意"
+    在`INSERT`查询中为了区分`mysql(...)`与带有列名列表的表名的表函数，你必须使用关键字`FUNCTION`或`TABLE FUNCTION`。查看如下示例。
 
 ## 用法示例 {#usage-example}
 
@@ -66,7 +74,7 @@ mysql> select * from test;
 1 row in set (0,00 sec)
 ```
 
-从ClickHouse中选择数据:
+从ClickHouse中查询数据:
 
 ``` sql
 SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123')
@@ -78,6 +86,21 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123')
 └────────┴──────────────┴───────┴────────────────┘
 ```
 
+替换和插入：
+
+```sql
+INSERT INTO FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 1) (int_id, float) VALUES (1, 3);
+INSERT INTO TABLE FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 0, 'UPDATE int_id = int_id + 1') (int_id, float) VALUES (1, 4);
+SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123');
+```
+
+```text
+┌─int_id─┬─float─┐
+│      1 │     3 │
+│      2 │     4 │
+└────────┴───────┘
+```
+
 ## 另请参阅 {#see-also}
 
 -   [该 ‘MySQL’ 表引擎](../../engines/table-engines/integrations/mysql.md)

programs/client/Client.cpp

@@ -430,6 +430,7 @@ private:
                {TokenType::ClosingRoundBracket, Replxx::Color::BROWN},
                {TokenType::OpeningSquareBracket, Replxx::Color::BROWN},
                {TokenType::ClosingSquareBracket, Replxx::Color::BROWN},
+               {TokenType::DoubleColon, Replxx::Color::BROWN},
                {TokenType::OpeningCurlyBrace, Replxx::Color::INTENSE},
                {TokenType::ClosingCurlyBrace, Replxx::Color::INTENSE},
 

programs/local/LocalServer.cpp

@@ -388,24 +388,32 @@ void LocalServer::processQueries()
     /// Use the same query_id (and thread group) for all queries
     CurrentThread::QueryScope query_scope_holder(context);
 
-    ///Set progress show
+    /// Set progress show
     need_render_progress = config().getBool("progress", false);
 
+    std::function<void()> finalize_progress;
     if (need_render_progress)
     {
+        /// Set progress callback, which can be run from multiple threads.
         context->setProgressCallback([&](const Progress & value)
         {
             /// Write progress only if progress was updated
             if (progress_indication.updateProgress(value))
                 progress_indication.writeProgress();
         });
+
+        /// Set finalizing callback for progress, which is called right before finalizing query output.
+        finalize_progress = [&]()
+        {
+            progress_indication.clearProgressOutput();
+        };
+
+        /// Set callback for file processing progress.
+        progress_indication.setFileProgressCallback(context);
     }
 
     bool echo_queries = config().hasOption("echo") || config().hasOption("verbose");
 
-    if (need_render_progress)
-        progress_indication.setFileProgressCallback(context);
-
     std::exception_ptr exception;
 
     for (const auto & query : queries)
@@ -425,7 +433,7 @@ void LocalServer::processQueries()
 
         try
         {
-            executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, context, {});
+            executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, context, {}, finalize_progress);
         }
         catch (...)
         {

programs/server/Server.cpp

@@ -477,17 +477,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
     CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision());
     CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
 
-    if (ThreadFuzzer::instance().isEffective())
-        LOG_WARNING(log, "ThreadFuzzer is enabled. Application will run slowly and unstable.");
-
-#if !defined(NDEBUG) || !defined(__OPTIMIZE__)
-    LOG_WARNING(log, "Server was built in debug mode. It will work slowly.");
-#endif
-
-#if defined(SANITIZER)
-    LOG_WARNING(log, "Server was built with sanitizer. It will work slowly.");
-#endif
-
     /** Context contains all that query execution is dependent:
       *  settings, available functions, data types, aggregate functions, databases, ...
       */
@@ -497,6 +486,18 @@ int Server::main(const std::vector<std::string> & /*args*/)
     global_context->makeGlobalContext();
     global_context->setApplicationType(Context::ApplicationType::SERVER);
 
+#if !defined(NDEBUG) || !defined(__OPTIMIZE__)
+    global_context->addWarningMessage("Server was built in debug mode. It will work slowly.");
+#endif
+
+if (ThreadFuzzer::instance().isEffective())
+    global_context->addWarningMessage("ThreadFuzzer is enabled. Application will run slowly and unstable.");
+
+#if defined(SANITIZER)
+    global_context->addWarningMessage("Server was built with sanitizer. It will work slowly.");
+#endif
+
+
     // Initialize global thread pool. Do it before we fetch configs from zookeeper
     // nodes (`from_zk`), because ZooKeeper interface uses the pool. We will
     // ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well.
@@ -552,8 +553,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
             if (ptrace(PTRACE_TRACEME, 0, nullptr, nullptr) == -1)
             {
                 /// Program is run under debugger. Modification of it's binary image is ok for breakpoints.
-                LOG_WARNING(log, "Server is run under debugger and its binary image is modified (most likely with breakpoints).",
+                global_context->addWarningMessage(
-                    calculated_binary_hash);
+                    fmt::format("Server is run under debugger and its binary image is modified (most likely with breakpoints).",
+                    calculated_binary_hash)
+                );
             }
             else
             {
@@ -636,7 +639,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
         }
         else
         {
-            LOG_WARNING(log, message);
+            global_context->addWarningMessage(message);
         }
     }
 

src/AggregateFunctions/AggregateFunctionBitwise.h

@@ -9,6 +9,14 @@
 
 #include <AggregateFunctions/IAggregateFunction.h>
 
+#if !defined(ARCADIA_BUILD)
+#    include <Common/config.h>
+#endif
+
+#if USE_EMBEDDED_COMPILER
+#    include <llvm/IR/IRBuilder.h>
+#    include <DataTypes/Native.h>
+#endif
 
 namespace DB
 {
@@ -21,6 +29,21 @@ struct AggregateFunctionGroupBitOrData
     T value = 0;
     static const char * name() { return "groupBitOr"; }
     void update(T x) { value |= x; }
+
+#if USE_EMBEDDED_COMPILER
+
+    static void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * value_ptr)
+    {
+        auto type = toNativeType<T>(builder);
+        builder.CreateStore(llvm::Constant::getNullValue(type), value_ptr);
+    }
+
+    static llvm::Value* compileUpdate(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs)
+    {
+        return builder.CreateOr(lhs, rhs);
+    }
+
+#endif
 };
 
 template <typename T>
@@ -29,6 +52,21 @@ struct AggregateFunctionGroupBitAndData
     T value = -1; /// Two's complement arithmetic, sign extension.
     static const char * name() { return "groupBitAnd"; }
     void update(T x) { value &= x; }
+
+#if USE_EMBEDDED_COMPILER
+
+    static void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * value_ptr)
+    {
+        auto type = toNativeType<T>(builder);
+        builder.CreateStore(llvm::ConstantInt::get(type, -1), value_ptr);
+    }
+
+    static llvm::Value* compileUpdate(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs)
+    {
+        return builder.CreateAnd(lhs, rhs);
+    }
+
+#endif
 };
 
 template <typename T>
@@ -37,6 +75,21 @@ struct AggregateFunctionGroupBitXorData
     T value = 0;
     static const char * name() { return "groupBitXor"; }
     void update(T x) { value ^= x; }
+
+#if USE_EMBEDDED_COMPILER
+
+    static void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * value_ptr)
+    {
+        auto type = toNativeType<T>(builder);
+        builder.CreateStore(llvm::Constant::getNullValue(type), value_ptr);
+    }
+
+    static llvm::Value* compileUpdate(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs)
+    {
+        return builder.CreateXor(lhs, rhs);
+    }
+
+#endif
 };
 
 
@@ -45,7 +98,7 @@ template <typename T, typename Data>
 class AggregateFunctionBitwise final : public IAggregateFunctionDataHelper<Data, AggregateFunctionBitwise<T, Data>>
 {
 public:
-    AggregateFunctionBitwise(const DataTypePtr & type)
+    explicit AggregateFunctionBitwise(const DataTypePtr & type)
         : IAggregateFunctionDataHelper<Data, AggregateFunctionBitwise<T, Data>>({type}, {}) {}
 
     String getName() const override { return Data::name(); }
@@ -81,6 +134,68 @@ public:
     {
         assert_cast<ColumnVector<T> &>(to).getData().push_back(this->data(place).value);
     }
+
+#if USE_EMBEDDED_COMPILER
+
+    bool isCompilable() const override
+    {
+        auto return_type = getReturnType();
+        return canBeNativeType(*return_type);
+    }
+
+    void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const override
+    {
+        llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
+
+        auto * return_type = toNativeType(b, getReturnType());
+        auto * value_ptr = b.CreatePointerCast(aggregate_data_ptr, return_type->getPointerTo());
+        Data::compileCreate(builder, value_ptr);
+    }
+
+    void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const DataTypes &, const std::vector<llvm::Value *> & argument_values) const override
+    {
+        llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
+
+        auto * return_type = toNativeType(b, getReturnType());
+
+        auto * value_ptr = b.CreatePointerCast(aggregate_data_ptr, return_type->getPointerTo());
+        auto * value = b.CreateLoad(return_type, value_ptr);
+
+        const auto & argument_value = argument_values[0];
+        auto * result_value = Data::compileUpdate(builder, value, argument_value);
+
+        b.CreateStore(result_value, value_ptr);
+    }
+
+    void compileMerge(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_dst_ptr, llvm::Value * aggregate_data_src_ptr) const override
+    {
+        llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
+
+        auto * return_type = toNativeType(b, getReturnType());
+
+        auto * value_dst_ptr = b.CreatePointerCast(aggregate_data_dst_ptr, return_type->getPointerTo());
+        auto * value_dst = b.CreateLoad(return_type, value_dst_ptr);
+
+        auto * value_src_ptr = b.CreatePointerCast(aggregate_data_src_ptr, return_type->getPointerTo());
+        auto * value_src = b.CreateLoad(return_type, value_src_ptr);
+
+        auto * result_value = Data::compileUpdate(builder, value_dst, value_src);
+
+        b.CreateStore(result_value, value_dst_ptr);
+    }
+
+    llvm::Value * compileGetResult(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const override
+    {
+        llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
+
+        auto * return_type = toNativeType(b, getReturnType());
+        auto * value_ptr = b.CreatePointerCast(aggregate_data_ptr, return_type->getPointerTo());
+
+        return b.CreateLoad(return_type, value_ptr);
+    }
+
+#endif
+
 };
 
 

src/AggregateFunctions/AggregateFunctionSequenceMatch.cpp

@@ -3,6 +3,7 @@
 #include <AggregateFunctions/AggregateFunctionSequenceMatch.h>
 
 #include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
 
 #include <common/range.h>

src/AggregateFunctions/AggregateFunctionSum.h

@@ -101,6 +101,24 @@ struct AggregateFunctionSumData
     {
         const auto * end = ptr + count;
 
+        if constexpr (
+            (is_integer_v<T> && !is_big_int_v<T>)
+            || (IsDecimalNumber<T> && !std::is_same_v<T, Decimal256> && !std::is_same_v<T, Decimal128>))
+        {
+            /// For integers we can vectorize the operation if we replace the null check using a multiplication (by 0 for null, 1 for not null)
+            /// https://quick-bench.com/q/MLTnfTvwC2qZFVeWHfOBR3U7a8I
+            T local_sum{};
+            while (ptr < end)
+            {
+                T multiplier = !*null_map;
+                Impl::add(local_sum, *ptr * multiplier);
+                ++ptr;
+                ++null_map;
+            }
+            Impl::add(sum, local_sum);
+            return;
+        }
+
         if constexpr (std::is_floating_point_v<T>)
         {
             constexpr size_t unroll_count = 128 / sizeof(T);

src/AggregateFunctions/AggregateFunctionUniq.cpp

@@ -4,6 +4,7 @@
 #include <AggregateFunctions/FactoryHelpers.h>
 
 #include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeTuple.h>
 #include <DataTypes/DataTypeUUID.h>
@@ -49,6 +50,8 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
             return res;
         else if (which.isDate())
             return std::make_shared<AggregateFunctionUniq<DataTypeDate::FieldType, Data>>(argument_types);
+        else if (which.isDate32())
+            return std::make_shared<AggregateFunctionUniq<DataTypeDate32::FieldType, Data>>(argument_types);
         else if (which.isDateTime())
             return std::make_shared<AggregateFunctionUniq<DataTypeDateTime::FieldType, Data>>(argument_types);
         else if (which.isStringOrFixedString())
@@ -95,6 +98,8 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
             return res;
         else if (which.isDate())
             return std::make_shared<AggregateFunctionUniq<DataTypeDate::FieldType, Data<DataTypeDate::FieldType>>>(argument_types);
+        else if (which.isDate32())
+            return std::make_shared<AggregateFunctionUniq<DataTypeDate32::FieldType, Data<DataTypeDate32::FieldType>>>(argument_types);
         else if (which.isDateTime())
             return std::make_shared<AggregateFunctionUniq<DataTypeDateTime::FieldType, Data<DataTypeDateTime::FieldType>>>(argument_types);
         else if (which.isStringOrFixedString())

src/AggregateFunctions/AggregateFunctionUniqCombined.cpp

@@ -6,6 +6,7 @@
 #include <Common/FieldVisitorConvertToNumber.h>
 
 #include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
 
 #include <functional>
@@ -51,6 +52,8 @@ namespace
                 return res;
             else if (which.isDate())
                 return std::make_shared<typename WithK<K, HashValueType>::template AggregateFunction<DataTypeDate::FieldType>>(argument_types, params);
+            else if (which.isDate32())
+                return std::make_shared<typename WithK<K, HashValueType>::template AggregateFunction<DataTypeDate32::FieldType>>(argument_types, params);
             else if (which.isDateTime())
                 return std::make_shared<typename WithK<K, HashValueType>::template AggregateFunction<DataTypeDateTime::FieldType>>(argument_types, params);
             else if (which.isStringOrFixedString())

src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp

@@ -3,6 +3,7 @@
 #include <AggregateFunctions/AggregateFunctionUniqUpTo.h>
 #include <Common/FieldVisitorConvertToNumber.h>
 #include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeFixedString.h>
@@ -61,6 +62,8 @@ AggregateFunctionPtr createAggregateFunctionUniqUpTo(const std::string & name, c
             return res;
         else if (which.isDate())
             return std::make_shared<AggregateFunctionUniqUpTo<DataTypeDate::FieldType>>(threshold, argument_types, params);
+        else if (which.isDate32())
+            return std::make_shared<AggregateFunctionUniqUpTo<DataTypeDate32::FieldType>>(threshold, argument_types, params);
         else if (which.isDateTime())
             return std::make_shared<AggregateFunctionUniqUpTo<DataTypeDateTime::FieldType>>(threshold, argument_types, params);
         else if (which.isStringOrFixedString())

src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp

@@ -4,6 +4,7 @@
 #include <AggregateFunctions/Helpers.h>
 #include <Core/Settings.h>
 #include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
 
 #include <common/range.h>

src/Common/Config/ConfigProcessor.cpp

@@ -298,11 +298,19 @@ void ConfigProcessor::doIncludesRecursive(
     {
         const auto * subst = attributes->getNamedItem(attr_name);
         attr_nodes[attr_name] = subst;
-        substs_count += static_cast<size_t>(subst == nullptr);
+        substs_count += static_cast<size_t>(subst != nullptr);
     }
 
-    if (substs_count < SUBSTITUTION_ATTRS.size() - 1) /// only one substitution is allowed
+    if (substs_count > 1) /// only one substitution is allowed
-        throw Poco::Exception("several substitutions attributes set for element <" + node->nodeName() + ">");
+        throw Poco::Exception("More than one substitution attribute is set for element <" + node->nodeName() + ">");
+
+    if (node->nodeName() == "include")
+    {
+        if (node->hasChildNodes())
+            throw Poco::Exception("<include> element must have no children");
+        if (substs_count == 0)
+            throw Poco::Exception("No substitution attributes set for element <include>, must have exactly one");
+    }
 
     /// Replace the original contents, not add to it.
     bool replace = attributes->getNamedItem("replace");
@@ -320,37 +328,57 @@ void ConfigProcessor::doIncludesRecursive(
             else if (throw_on_bad_incl)
                 throw Poco::Exception(error_msg + name);
             else
+            {
+                if (node->nodeName() == "include")
+                    node->parentNode()->removeChild(node);
+
                 LOG_WARNING(log, "{}{}", error_msg, name);
+            }
         }
         else
         {
-            Element & element = dynamic_cast<Element &>(*node);
+            /// Replace the whole node not just contents.
-
+            if (node->nodeName() == "include")
-            for (const auto & attr_name : SUBSTITUTION_ATTRS)
-                element.removeAttribute(attr_name);
-
-            if (replace)
             {
-                while (Node * child = node->firstChild())
+                const NodeListPtr children = node_to_include->childNodes();
-                    node->removeChild(child);
+                for (size_t i = 0, size = children->length(); i < size; ++i)
+                {
+                    NodePtr new_node = config->importNode(children->item(i), true);
+                    node->parentNode()->insertBefore(new_node, node);
+                }
 
-                element.removeAttribute("replace");
+                node->parentNode()->removeChild(node);
             }
-
+            else
-            const NodeListPtr children = node_to_include->childNodes();
-            for (size_t i = 0, size = children->length(); i < size; ++i)
             {
-                NodePtr new_node = config->importNode(children->item(i), true);
+                Element & element = dynamic_cast<Element &>(*node);
-                node->appendChild(new_node);
-            }
 
-            const NamedNodeMapPtr from_attrs = node_to_include->attributes();
+                for (const auto & attr_name : SUBSTITUTION_ATTRS)
-            for (size_t i = 0, size = from_attrs->length(); i < size; ++i)
+                    element.removeAttribute(attr_name);
-            {
-                element.setAttributeNode(dynamic_cast<Attr *>(config->importNode(from_attrs->item(i), true)));
-            }
 
-            included_something = true;
+                if (replace)
+                {
+                    while (Node * child = node->firstChild())
+                        node->removeChild(child);
+
+                    element.removeAttribute("replace");
+                }
+
+                const NodeListPtr children = node_to_include->childNodes();
+                for (size_t i = 0, size = children->length(); i < size; ++i)
+                {
+                    NodePtr new_node = config->importNode(children->item(i), true);
+                    node->appendChild(new_node);
+                }
+
+                const NamedNodeMapPtr from_attrs = node_to_include->attributes();
+                for (size_t i = 0, size = from_attrs->length(); i < size; ++i)
+                {
+                    element.setAttributeNode(dynamic_cast<Attr *>(config->importNode(from_attrs->item(i), true)));
+                }
+
+                included_something = true;
+            }
         }
     };
 

src/Common/Config/configReadClient.cpp

@@ -10,16 +10,10 @@ namespace fs = std::filesystem;
 namespace DB
 {
 
-/// Checks if file exists without throwing an exception but with message in console.
+bool safeFsExists(const String & path)
-bool safeFsExists(const auto & path)
 {
     std::error_code ec;
-    bool res = fs::exists(path, ec);
+    return fs::exists(path, ec);
-    if (ec)
-    {
-        std::cerr << "Can't check '" << path << "': [" << ec.value() << "] "  << ec.message() << std::endl;
-    }
-    return res;
 };
 
 bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::string & home_path)

src/Common/HashTable/StringHashTable.h

@@ -237,7 +237,12 @@ public:
     // 1. Always memcpy 8 times bytes
     // 2. Use switch case extension to generate fast dispatching table
     // 3. Funcs are named callables that can be force_inlined
+    //
     // NOTE: It relies on Little Endianness
+    //
+    // NOTE: It requires padded to 8 bytes keys (IOW you cannot pass
+    // std::string here, but you can pass i.e. ColumnString::getDataAt()),
+    // since it copies 8 bytes at a time.
     template <typename Self, typename KeyHolder, typename Func>
     static auto ALWAYS_INLINE dispatch(Self & self, KeyHolder && key_holder, Func && func)
     {

src/Common/ProfileEvents.cpp

@@ -22,10 +22,6 @@
     M(WriteBufferFromFileDescriptorWrite, "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.") \
     M(WriteBufferFromFileDescriptorWriteFailed, "Number of times the write (write/pwrite) to a file descriptor have failed.") \
     M(WriteBufferFromFileDescriptorWriteBytes, "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.") \
-    M(ReadBufferAIORead, "") \
-    M(ReadBufferAIOReadBytes, "") \
-    M(WriteBufferAIOWrite, "") \
-    M(WriteBufferAIOWriteBytes, "") \
     M(ReadCompressedBytes, "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).") \
     M(CompressedReadBufferBlocks, "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).") \
     M(CompressedReadBufferBytes, "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).") \
@@ -34,6 +30,10 @@
     M(UncompressedCacheWeightLost, "") \
     M(MMappedFileCacheHits, "") \
     M(MMappedFileCacheMisses, "") \
+    M(AIOWrite, "Number of writes with Linux or FreeBSD AIO interface") \
+    M(AIOWriteBytes, "Number of bytes written with Linux or FreeBSD AIO interface") \
+    M(AIORead, "Number of reads with Linux or FreeBSD AIO interface") \
+    M(AIOReadBytes, "Number of bytes read with Linux or FreeBSD AIO interface") \
     M(IOBufferAllocs, "") \
     M(IOBufferAllocBytes, "") \
     M(ArenaAllocChunks, "") \
@@ -43,8 +43,8 @@
     M(MarkCacheHits, "") \
     M(MarkCacheMisses, "") \
     M(CreatedReadBufferOrdinary, "") \
-    M(CreatedReadBufferAIO, "") \
+    M(CreatedReadBufferDirectIO, "") \
-    M(CreatedReadBufferAIOFailed, "") \
+    M(CreatedReadBufferDirectIOFailed, "") \
     M(CreatedReadBufferMMap, "") \
     M(CreatedReadBufferMMapFailed, "") \
     M(DiskReadElapsedMicroseconds, "Total time spent waiting for read syscall. This include reads from page cache.") \

src/Common/ProgressIndication.cpp

@@ -4,9 +4,6 @@
 #include <Common/UnicodeBar.h>
 #include <Databases/DatabaseMemory.h>
 
-/// FIXME: progress bar in clickhouse-local needs to be cleared after query execution
-///        - same as it is now in clickhouse-client. Also there is no writeFinalProgress call
-///        in clickhouse-local.
 
 namespace DB
 {

src/Common/ZooKeeper/ZooKeeperCommon.h

@@ -45,6 +45,8 @@ struct ZooKeeperRequest : virtual Request
     /// If the request was sent and we didn't get the response and the error happens, then we cannot be sure was it processed or not.
     bool probably_sent = false;
 
+    bool restored_from_zookeeper_log = false;
+
     ZooKeeperRequest() = default;
     ZooKeeperRequest(const ZooKeeperRequest &) = default;
     virtual ~ZooKeeperRequest() override = default;
@@ -172,6 +174,9 @@ struct ZooKeeperCloseResponse final : ZooKeeperResponse
 
 struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest
 {
+    /// used only during restore from zookeeper log
+    int32_t parent_cversion = -1;
+
     ZooKeeperCreateRequest() = default;
     explicit ZooKeeperCreateRequest(const CreateRequest & base) : CreateRequest(base) {}
 
@@ -183,9 +188,6 @@ struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest
     bool isReadRequest() const override { return false; }
 
     size_t bytesSize() const override { return CreateRequest::bytesSize() + sizeof(xid) + sizeof(has_watch); }
-
-    /// During recovery from log we don't rehash ACLs
-    bool need_to_hash_acls = true;
 };
 
 struct ZooKeeperCreateResponse final : CreateResponse, ZooKeeperResponse
@@ -362,8 +364,6 @@ struct ZooKeeperSetACLRequest final : SetACLRequest, ZooKeeperRequest
     bool isReadRequest() const override { return false; }
 
     size_t bytesSize() const override { return SetACLRequest::bytesSize() + sizeof(xid); }
-
-    bool need_to_hash_acls = true;
 };
 
 struct ZooKeeperSetACLResponse final : SetACLResponse, ZooKeeperResponse

src/Compression/CompressedReadBufferFromFile.cpp

@@ -47,13 +47,13 @@ CompressedReadBufferFromFile::CompressedReadBufferFromFile(std::unique_ptr<ReadB
 CompressedReadBufferFromFile::CompressedReadBufferFromFile(
     const std::string & path,
     size_t estimated_size,
-    size_t aio_threshold,
+    size_t direct_io_threshold,
     size_t mmap_threshold,
     MMappedFileCache * mmap_cache,
     size_t buf_size,
     bool allow_different_codecs_)
     : BufferWithOwnMemory<ReadBuffer>(0)
-    , p_file_in(createReadBufferFromFileBase(path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size))
+    , p_file_in(createReadBufferFromFileBase(path, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache, buf_size))
     , file_in(*p_file_in)
 {
     compressed_in = &file_in;

src/Compression/CompressedReadBufferFromFile.h

@@ -33,7 +33,7 @@ public:
     CompressedReadBufferFromFile(std::unique_ptr<ReadBufferFromFileBase> buf, bool allow_different_codecs_ = false);
 
     CompressedReadBufferFromFile(
-        const std::string & path, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache,
+        const std::string & path, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache,
         size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, bool allow_different_codecs_ = false);
 
     void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);

src/Coordination/KeeperStorage.cpp

@@ -267,13 +267,12 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest
             }
             else
             {
-
                 auto & session_auth_ids = storage.session_and_auth[session_id];
 
                 KeeperStorage::Node created_node;
 
                 Coordination::ACLs node_acls;
-                if (!fixupACL(request.acls, session_auth_ids, node_acls, request.need_to_hash_acls))
+                if (!fixupACL(request.acls, session_auth_ids, node_acls, !request.restored_from_zookeeper_log))
                 {
                     response.error = Coordination::Error::ZINVALIDACL;
                     return {response_ptr, {}};
@@ -307,16 +306,28 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest
                     path_created += seq_num_str.str();
                 }
 
+                int32_t parent_cversion = request.parent_cversion;
                 auto child_path = getBaseName(path_created);
                 int64_t prev_parent_zxid;
-                container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid] (KeeperStorage::Node & parent)
+                int32_t prev_parent_cversion;
+                container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid,
+                                                    parent_cversion, &prev_parent_cversion] (KeeperStorage::Node & parent)
                 {
+
+                    parent.children.insert(child_path);
+                    prev_parent_cversion = parent.stat.cversion;
+                    prev_parent_zxid = parent.stat.pzxid;
+
                     /// Increment sequential number even if node is not sequential
                     ++parent.seq_num;
-                    parent.children.insert(child_path);
+
-                    ++parent.stat.cversion;
+                    if (parent_cversion == -1)
-                    prev_parent_zxid = parent.stat.pzxid;
+                        ++parent.stat.cversion;
-                    parent.stat.pzxid = zxid;
+                    else if (parent_cversion > parent.stat.cversion)
+                        parent.stat.cversion = parent_cversion;
+
+                    if (zxid > parent.stat.pzxid)
+                        parent.stat.pzxid = zxid;
                     ++parent.stat.numChildren;
                 });
 
@@ -326,7 +337,7 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest
                 if (request.is_ephemeral)
                     ephemerals[session_id].emplace(path_created);
 
-                undo = [&storage, prev_parent_zxid, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id]
+                undo = [&storage, prev_parent_zxid, prev_parent_cversion, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id]
                 {
                     storage.container.erase(path_created);
                     storage.acl_map.removeUsage(acl_id);
@@ -334,11 +345,11 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest
                     if (is_ephemeral)
                         storage.ephemerals[session_id].erase(path_created);
 
-                    storage.container.updateValue(parent_path, [child_path, prev_parent_zxid] (KeeperStorage::Node & undo_parent)
+                    storage.container.updateValue(parent_path, [child_path, prev_parent_zxid, prev_parent_cversion] (KeeperStorage::Node & undo_parent)
                     {
-                        --undo_parent.stat.cversion;
                         --undo_parent.stat.numChildren;
                         --undo_parent.seq_num;
+                        undo_parent.stat.cversion = prev_parent_cversion;
                         undo_parent.stat.pzxid = prev_parent_zxid;
                         undo_parent.children.erase(child_path);
                     });
@@ -394,6 +405,24 @@ struct KeeperStorageGetRequest final : public KeeperStorageRequest
     }
 };
 
+namespace
+{
+    /// Garbage required to apply log to "fuzzy" zookeeper snapshot
+    void updateParentPzxid(const std::string & child_path, int64_t zxid, KeeperStorage::Container & container)
+    {
+        auto parent_path = parentPath(child_path);
+        auto parent_it = container.find(parent_path);
+        if (parent_it != container.end())
+        {
+            container.updateValue(parent_path, [zxid](KeeperStorage::Node & parent)
+            {
+                if (parent.stat.pzxid < zxid)
+                    parent.stat.pzxid = zxid;
+            });
+        }
+    }
+}
+
 struct KeeperStorageRemoveRequest final : public KeeperStorageRequest
 {
     bool checkAuth(KeeperStorage & storage, int64_t session_id) const override
@@ -412,7 +441,7 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest
     }
 
     using KeeperStorageRequest::KeeperStorageRequest;
-    std::pair<Coordination::ZooKeeperResponsePtr, Undo> process(KeeperStorage & storage, int64_t /*zxid*/, int64_t /*session_id*/) const override
+    std::pair<Coordination::ZooKeeperResponsePtr, Undo> process(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/) const override
     {
         auto & container = storage.container;
         auto & ephemerals = storage.ephemerals;
@@ -425,6 +454,8 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest
         auto it = container.find(request.path);
         if (it == container.end())
         {
+            if (request.restored_from_zookeeper_log)
+                updateParentPzxid(request.path, zxid, container);
             response.error = Coordination::Error::ZNONODE;
         }
         else if (request.version != -1 && request.version != it->value.stat.version)
@@ -437,6 +468,9 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest
         }
         else
         {
+            if (request.restored_from_zookeeper_log)
+                updateParentPzxid(request.path, zxid, container);
+
             auto prev_node = it->value;
             if (prev_node.stat.ephemeralOwner != 0)
             {
@@ -719,7 +753,7 @@ struct KeeperStorageSetACLRequest final : public KeeperStorageRequest
             auto & session_auth_ids = storage.session_and_auth[session_id];
             Coordination::ACLs node_acls;
 
-            if (!fixupACL(request.acls, session_auth_ids, node_acls, request.need_to_hash_acls))
+            if (!fixupACL(request.acls, session_auth_ids, node_acls, !request.restored_from_zookeeper_log))
             {
                 response.error = Coordination::Error::ZINVALIDACL;
                 return {response_ptr, {}};

src/Coordination/ZooKeeperDataReader.cpp

@@ -174,7 +174,22 @@ void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::st
 
     LOG_INFO(log, "Deserializing data from snapshot");
     int64_t zxid_from_nodes = deserializeStorageData(storage, reader, log);
-    storage.zxid = std::max(zxid, zxid_from_nodes);
+    /// In ZooKeeper Snapshots can contain inconsistent state of storage. They call
+    /// this inconsistent state "fuzzy". So it's guaranteed that snapshot contain all
+    /// records up to zxid from snapshot name and also some records for future.
+    /// But it doesn't mean that we have just some state of storage from future (like zxid + 100 log records).
+    /// We have incorrect state of storage where some random log entries from future were applied....
+    ///
+    /// In ZooKeeper they say that their transactions log is idempotent and can be applied to "fuzzy" state as is.
+    /// It's true but there is no any general invariant which produces this property. They just have ad-hoc "if's" which detects
+    /// "fuzzy" state inconsistencies and apply log records in special way. Several examples:
+    /// https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java#L453-L463
+    /// https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java#L476-L480
+    /// https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java#L547-L549
+    if (zxid_from_nodes > zxid)
+        LOG_WARNING(log, "ZooKeeper snapshot was in inconsistent (fuzzy) state. Will try to apply log. ZooKeeper create non fuzzy snapshot with restart. You can just restart ZooKeeper server and get consistent version.");
+
+    storage.zxid = zxid;
 
     LOG_INFO(log, "Finished, snapshot ZXID {}", storage.zxid);
 }
@@ -210,16 +225,18 @@ void deserializeLogMagic(ReadBuffer & in)
 
     static constexpr int32_t LOG_HEADER = 1514884167; /// "ZKLG"
     if (magic_header != LOG_HEADER)
-        throw Exception(ErrorCodes::CORRUPTED_DATA ,"Incorrect magic header in file, expected {}, got {}", LOG_HEADER, magic_header);
+        throw Exception(ErrorCodes::CORRUPTED_DATA, "Incorrect magic header in file, expected {}, got {}", LOG_HEADER, magic_header);
 
     if (version != 2)
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED,"Cannot deserialize ZooKeeper data other than version 2, got version {}", version);
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot deserialize ZooKeeper data other than version 2, got version {}", version);
 }
 
 
-/// For some reason zookeeper stores slightly different records in log then
+/// ZooKeeper transactions log differs from requests. The main reason: to store records in log
-/// requests. For example:
+/// in some "finalized" state (for example with concrete versions).
-///  class CreateTxn {
+///
+/// Example:
+/// class CreateTxn {
 ///      ustring path;
 ///      buffer data;
 ///      vector<org.apache.zookeeper.data.ACL> acl;
@@ -289,10 +306,9 @@ Coordination::ZooKeeperRequestPtr deserializeCreateTxn(ReadBuffer & in)
     Coordination::read(result->data, in);
     Coordination::read(result->acls, in);
     Coordination::read(result->is_ephemeral, in);
-    result->need_to_hash_acls = false;
+    Coordination::read(result->parent_cversion, in);
-    /// How we should use it? It should just increment on request execution
+
-    int32_t parent_c_version;
+    result->restored_from_zookeeper_log = true;
-    Coordination::read(parent_c_version, in);
     return result;
 }
 
@@ -300,6 +316,7 @@ Coordination::ZooKeeperRequestPtr deserializeDeleteTxn(ReadBuffer & in)
 {
     std::shared_ptr<Coordination::ZooKeeperRemoveRequest> result = std::make_shared<Coordination::ZooKeeperRemoveRequest>();
     Coordination::read(result->path, in);
+    result->restored_from_zookeeper_log = true;
     return result;
 }
 
@@ -309,6 +326,7 @@ Coordination::ZooKeeperRequestPtr deserializeSetTxn(ReadBuffer & in)
     Coordination::read(result->path, in);
     Coordination::read(result->data, in);
     Coordination::read(result->version, in);
+    result->restored_from_zookeeper_log = true;
     /// It stores version + 1 (which should be, not for request)
     result->version -= 1;
 
@@ -320,6 +338,7 @@ Coordination::ZooKeeperRequestPtr deserializeCheckVersionTxn(ReadBuffer & in)
     std::shared_ptr<Coordination::ZooKeeperCheckRequest> result = std::make_shared<Coordination::ZooKeeperCheckRequest>();
     Coordination::read(result->path, in);
     Coordination::read(result->version, in);
+    result->restored_from_zookeeper_log = true;
     return result;
 }
 
@@ -329,14 +348,19 @@ Coordination::ZooKeeperRequestPtr deserializeCreateSession(ReadBuffer & in)
     int32_t timeout;
     Coordination::read(timeout, in);
     result->session_timeout_ms = timeout;
+    result->restored_from_zookeeper_log = true;
     return result;
 }
 
-Coordination::ZooKeeperRequestPtr deserializeCloseSession(ReadBuffer & in)
+Coordination::ZooKeeperRequestPtr deserializeCloseSession(ReadBuffer & in, bool empty)
 {
     std::shared_ptr<Coordination::ZooKeeperCloseRequest> result = std::make_shared<Coordination::ZooKeeperCloseRequest>();
-    std::vector<std::string> data;
+    if (!empty)
-    Coordination::read(data, in);
+    {
+        std::vector<std::string> data;
+        Coordination::read(data, in);
+    }
+    result->restored_from_zookeeper_log = true;
     return result;
 }
 
@@ -356,14 +380,14 @@ Coordination::ZooKeeperRequestPtr deserializeSetACLTxn(ReadBuffer & in)
     Coordination::read(result->version, in);
     /// It stores version + 1 (which should be, not for request)
     result->version -= 1;
-    result->need_to_hash_acls = false;
+    result->restored_from_zookeeper_log = true;
 
     return result;
 }
 
 Coordination::ZooKeeperRequestPtr deserializeMultiTxn(ReadBuffer & in);
 
-Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtxn)
+Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtxn, int64_t txn_length = 0)
 {
     int32_t type;
     Coordination::read(type, in);
@@ -372,6 +396,11 @@ Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtx
     if (subtxn)
         Coordination::read(sub_txn_length, in);
 
+    bool empty_txn = !subtxn && txn_length == 32; /// Possible for old-style CloseTxn's
+
+    if (empty_txn && type != -11)
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Empty non close-session transaction found");
+
     int64_t in_count_before = in.count();
 
     switch (type)
@@ -398,7 +427,7 @@ Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtx
             result = deserializeCreateSession(in);
             break;
         case -11:
-            result = deserializeCloseSession(in);
+            result = deserializeCloseSession(in, empty_txn);
             break;
         case -1:
             result = deserializeErrorTxn(in);
@@ -442,7 +471,7 @@ bool hasErrorsInMultiRequest(Coordination::ZooKeeperRequestPtr request)
     if (request == nullptr)
         return true;
 
-    for (const auto & subrequest : dynamic_cast<Coordination::ZooKeeperMultiRequest *>(request.get())->requests) //-V522
+    for (const auto & subrequest : dynamic_cast<Coordination::ZooKeeperMultiRequest *>(request.get())->requests) // -V522
         if (subrequest == nullptr)
             return true;
     return false;
@@ -470,7 +499,7 @@ bool deserializeTxn(KeeperStorage & storage, ReadBuffer & in, Poco::Logger * /*l
     int64_t time;
     Coordination::read(time, in);
 
-    Coordination::ZooKeeperRequestPtr request = deserializeTxnImpl(in, false);
+    Coordination::ZooKeeperRequestPtr request = deserializeTxnImpl(in, false, txn_len);
 
     /// Skip all other bytes
     int64_t bytes_read = in.count() - count_before;

src/Core/ExternalResultDescription.cpp

@@ -62,6 +62,8 @@ void ExternalResultDescription::init(const Block & sample_block_)
             types.emplace_back(ValueType::vtString, is_nullable);
         else if (which.isDate())
             types.emplace_back(ValueType::vtDate, is_nullable);
+        else if (which.isDate32())
+            types.emplace_back(ValueType::vtDate32, is_nullable);
         else if (which.isDateTime())
             types.emplace_back(ValueType::vtDateTime, is_nullable);
         else if (which.isUUID())

src/Core/ExternalResultDescription.h

@@ -26,6 +26,7 @@ struct ExternalResultDescription
         vtEnum16,
         vtString,
         vtDate,
+        vtDate32,
         vtDateTime,
         vtUUID,
         vtDateTime64,

src/Core/PostgreSQL/insertPostgreSQLValue.cpp

@@ -89,6 +89,9 @@ void insertPostgreSQLValue(
         case ExternalResultDescription::ValueType::vtDate:
             assert_cast<ColumnUInt16 &>(column).insertValue(UInt16{LocalDate{std::string(value)}.getDayNum()});
             break;
+        case ExternalResultDescription::ValueType::vtDate32:
+            assert_cast<ColumnInt32 &>(column).insertValue(Int32{LocalDate{std::string(value)}.getExtenedDayNum()});
+            break;
         case ExternalResultDescription::ValueType::vtDateTime:
         {
             ReadBufferFromString in(value);

src/Core/Settings.h

@@ -108,7 +108,7 @@ class IColumn;
     M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \
     M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \
     M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \
-    M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \
+    M(UInt64, min_count_to_compile_aggregate_expression, 0, "The number of identical aggregate expressions before they are JIT-compiled", 0) \
     M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \
     M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \
     M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \

src/Core/Types.h

@@ -39,6 +39,7 @@ enum class TypeIndex
     Float32,
     Float64,
     Date,
+    Date32,
     DateTime,
     DateTime64,
     String,
@@ -257,6 +258,7 @@ inline constexpr const char * getTypeName(TypeIndex idx)
         case TypeIndex::Float32:    return "Float32";
         case TypeIndex::Float64:    return "Float64";
         case TypeIndex::Date:       return "Date";
+        case TypeIndex::Date32:     return "Date32";
         case TypeIndex::DateTime:   return "DateTime";
         case TypeIndex::DateTime64: return "DateTime64";
         case TypeIndex::String:     return "String";

src/Core/callOnTypeIndex.h

@@ -192,6 +192,7 @@ bool callOnIndexAndDataType(TypeIndex number, F && f, ExtraArgs && ... args)
         case TypeIndex::Decimal256:     return f(TypePair<DataTypeDecimal<Decimal256>, T>(), std::forward<ExtraArgs>(args)...);
 
         case TypeIndex::Date:           return f(TypePair<DataTypeDate, T>(), std::forward<ExtraArgs>(args)...);
+        case TypeIndex::Date32:         return f(TypePair<DataTypeDate, T>(), std::forward<ExtraArgs>(args)...);
         case TypeIndex::DateTime:       return f(TypePair<DataTypeDateTime, T>(), std::forward<ExtraArgs>(args)...);
         case TypeIndex::DateTime64:     return f(TypePair<DataTypeDateTime64, T>(), std::forward<ExtraArgs>(args)...);
 

src/DataTypes/DataTypeDate32.cpp

@@ -0,0 +1,23 @@
+#include <DataTypes/DataTypeDate32.h>
+#include <DataTypes/DataTypeFactory.h>
+#include <DataTypes/Serializations/SerializationDate32.h>
+
+namespace DB
+{
+bool DataTypeDate32::equals(const IDataType & rhs) const
+{
+    return typeid(rhs) == typeid(*this);
+}
+
+SerializationPtr DataTypeDate32::doGetDefaultSerialization() const
+{
+    return std::make_shared<SerializationDate32>();
+}
+
+void registerDataTypeDate32(DataTypeFactory & factory)
+{
+    factory.registerSimpleDataType(
+        "Date32", [] { return DataTypePtr(std::make_shared<DataTypeDate32>()); }, DataTypeFactory::CaseInsensitive);
+}
+
+}

src/DataTypes/DataTypeDate32.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <DataTypes/DataTypeNumberBase.h>
+
+namespace DB
+{
+class DataTypeDate32 final : public DataTypeNumberBase<Int32>
+{
+public:
+    static constexpr auto family_name = "Date32";
+
+    TypeIndex getTypeId() const override { return TypeIndex::Date32; }
+    const char * getFamilyName() const override { return family_name; }
+
+    bool canBeUsedAsVersion() const override { return true; }
+    bool canBeInsideNullable() const override { return true; }
+
+    bool equals(const IDataType & rhs) const override;
+
+protected:
+    SerializationPtr doGetDefaultSerialization() const override;
+};
+}

src/DataTypes/DataTypeFactory.cpp

@@ -194,6 +194,7 @@ DataTypeFactory::DataTypeFactory()
     registerDataTypeNumbers(*this);
     registerDataTypeDecimal(*this);
     registerDataTypeDate(*this);
+    registerDataTypeDate32(*this);
     registerDataTypeDateTime(*this);
     registerDataTypeString(*this);
     registerDataTypeFixedString(*this);

src/DataTypes/DataTypeFactory.h

@@ -69,6 +69,7 @@ private:
 void registerDataTypeNumbers(DataTypeFactory & factory);
 void registerDataTypeDecimal(DataTypeFactory & factory);
 void registerDataTypeDate(DataTypeFactory & factory);
+void registerDataTypeDate32(DataTypeFactory & factory);
 void registerDataTypeDateTime(DataTypeFactory & factory);
 void registerDataTypeString(DataTypeFactory & factory);
 void registerDataTypeFixedString(DataTypeFactory & factory);

src/DataTypes/DataTypeLowCardinality.cpp

@@ -78,6 +78,8 @@ MutableColumnUniquePtr DataTypeLowCardinality::createColumnUniqueImpl(const IDat
         return creator(static_cast<ColumnFixedString *>(nullptr));
     else if (which.isDate())
         return creator(static_cast<ColumnVector<UInt16> *>(nullptr));
+    else if (which.isDate32())
+        return creator(static_cast<ColumnVector<Int32> *>(nullptr));
     else if (which.isDateTime())
         return creator(static_cast<ColumnVector<UInt32> *>(nullptr));
     else if (which.isUUID())

src/DataTypes/DataTypeLowCardinalityHelpers.cpp

@@ -1,11 +1,13 @@
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnConst.h>
 #include <Columns/ColumnTuple.h>
+#include <Columns/ColumnMap.h>
 #include <Columns/ColumnLowCardinality.h>
 
 #include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeTuple.h>
+#include <DataTypes/DataTypeMap.h>
 
 #include <Common/assert_cast.h>
 
@@ -39,6 +41,11 @@ DataTypePtr recursiveRemoveLowCardinality(const DataTypePtr & type)
             return std::make_shared<DataTypeTuple>(elements);
     }
 
+    if (const auto * map_type = typeid_cast<const DataTypeMap *>(type.get()))
+    {
+        return std::make_shared<DataTypeMap>(recursiveRemoveLowCardinality(map_type->getKeyType()), recursiveRemoveLowCardinality(map_type->getValueType()));
+    }
+
     if (const auto * low_cardinality_type = typeid_cast<const DataTypeLowCardinality *>(type.get()))
         return low_cardinality_type->getDictionaryType();
 
@@ -78,6 +85,16 @@ ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column)
         return ColumnTuple::create(columns);
     }
 
+    if (const auto * column_map = typeid_cast<const ColumnMap *>(column.get()))
+    {
+        const auto & nested = column_map->getNestedColumnPtr();
+        auto nested_no_lc = recursiveRemoveLowCardinality(nested);
+        if (nested.get() == nested_no_lc.get())
+            return column;
+
+        return ColumnMap::create(nested_no_lc);
+    }
+
     if (const auto * column_low_cardinality = typeid_cast<const ColumnLowCardinality *>(column.get()))
         return column_low_cardinality->convertToFullColumn();
 

src/DataTypes/DataTypeMap.cpp

@@ -7,6 +7,7 @@
 #include <DataTypes/DataTypeMap.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeTuple.h>
+#include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/Serializations/SerializationMap.h>
 #include <Parsers/IAST.h>
@@ -53,12 +54,24 @@ DataTypeMap::DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & valu
 
 void DataTypeMap::assertKeyType() const
 {
-    if (!key_type->isValueRepresentedByInteger()
+    bool type_error = false;
+    if (key_type->getTypeId() == TypeIndex::LowCardinality)
+    {
+        const auto & low_cardinality_data_type = assert_cast<const DataTypeLowCardinality &>(*key_type);
+        if (!isStringOrFixedString(*(low_cardinality_data_type.getDictionaryType())))
+            type_error = true;
+    }
+    else if (!key_type->isValueRepresentedByInteger()
         && !isStringOrFixedString(*key_type)
         && !WhichDataType(key_type).isNothing()
         && !WhichDataType(key_type).isUUID())
+    {
+        type_error = true;
+    }
+
+    if (type_error)
         throw Exception(ErrorCodes::BAD_ARGUMENTS,
-            "Type of Map key must be a type, that can be represented by integer or string or UUID,"
+            "Type of Map key must be a type, that can be represented by integer or String or FixedString (possibly LowCardinality) or UUID,"
             " but {} given", key_type->getName());
 }
 

src/DataTypes/IDataType.h

@@ -322,8 +322,10 @@ struct WhichDataType
     constexpr bool isEnum() const { return isEnum8() || isEnum16(); }
 
     constexpr bool isDate() const { return idx == TypeIndex::Date; }
+    constexpr bool isDate32() const { return idx == TypeIndex::Date32; }
     constexpr bool isDateTime() const { return idx == TypeIndex::DateTime; }
     constexpr bool isDateTime64() const { return idx == TypeIndex::DateTime64; }
+    constexpr bool isDateOrDate32() const { return isDate() || isDate32(); }
 
     constexpr bool isString() const { return idx == TypeIndex::String; }
     constexpr bool isFixedString() const { return idx == TypeIndex::FixedString; }
@@ -347,6 +349,10 @@ struct WhichDataType
 template <typename T>
 inline bool isDate(const T & data_type) { return WhichDataType(data_type).isDate(); }
 template <typename T>
+inline bool isDate32(const T & data_type) { return WhichDataType(data_type).isDate32(); }
+template <typename T>
+inline bool isDateOrDate32(const T & data_type) { return WhichDataType(data_type).isDateOrDate32(); }
+template <typename T>
 inline bool isDateTime(const T & data_type) { return WhichDataType(data_type).isDateTime(); }
 template <typename T>
 inline bool isDateTime64(const T & data_type) { return WhichDataType(data_type).isDateTime64(); }

src/DataTypes/Native.h

@@ -29,7 +29,7 @@ namespace ErrorCodes
 static inline bool typeIsSigned(const IDataType & type)
 {
     WhichDataType data_type(type);
-    return data_type.isNativeInt() || data_type.isFloat();
+    return data_type.isNativeInt() || data_type.isFloat() || data_type.isEnum();
 }
 
 static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDataType & type)
@@ -57,6 +57,10 @@ static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDa
         return builder.getFloatTy();
     else if (data_type.isFloat64())
         return builder.getDoubleTy();
+    else if (data_type.isEnum8())
+        return builder.getInt8Ty();
+    else if (data_type.isEnum16())
+        return builder.getInt16Ty();
 
     return nullptr;
 }
@@ -109,7 +113,7 @@ static inline bool canBeNativeType(const IDataType & type)
         return canBeNativeType(*data_type_nullable.getNestedType());
     }
 
-    return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate();
+    return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate() || data_type.isEnum();
 }
 
 static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const DataTypePtr & type)
@@ -266,7 +270,7 @@ static inline llvm::Constant * getColumnNativeValue(llvm::IRBuilderBase & builde
     {
         return llvm::ConstantInt::get(type, column.getUInt(index));
     }
-    else if (column_data_type.isNativeInt())
+    else if (column_data_type.isNativeInt() || column_data_type.isEnum())
     {
         return llvm::ConstantInt::get(type, column.getInt(index));
     }

src/DataTypes/Serializations/SerializationDate32.cpp

@@ -0,0 +1,78 @@
+#include <DataTypes/Serializations/SerializationDate32.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+
+#include <Columns/ColumnsNumber.h>
+
+#include <Common/assert_cast.h>
+
+namespace DB
+{
+void SerializationDate32::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
+{
+    writeDateText(ExtendedDayNum(assert_cast<const ColumnInt32 &>(column).getData()[row_num]), ostr);
+}
+
+void SerializationDate32::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
+{
+    deserializeTextEscaped(column, istr, settings);
+}
+
+void SerializationDate32::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+{
+    ExtendedDayNum x;
+    readDateText(x, istr);
+    assert_cast<ColumnInt32 &>(column).getData().push_back(x);
+}
+
+void SerializationDate32::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
+{
+    serializeText(column, row_num, ostr, settings);
+}
+
+void SerializationDate32::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
+{
+    writeChar('\'', ostr);
+    serializeText(column, row_num, ostr, settings);
+    writeChar('\'', ostr);
+}
+
+void SerializationDate32::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+{
+    ExtendedDayNum x;
+    assertChar('\'', istr);
+    readDateText(x, istr);
+    assertChar('\'', istr);
+    assert_cast<ColumnInt32 &>(column).getData().push_back(x);    /// It's important to do this at the end - for exception safety.
+}
+
+void SerializationDate32::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
+{
+    writeChar('"', ostr);
+    serializeText(column, row_num, ostr, settings);
+    writeChar('"', ostr);
+}
+
+void SerializationDate32::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+{
+    ExtendedDayNum x;
+    assertChar('"', istr);
+    readDateText(x, istr);
+    assertChar('"', istr);
+    assert_cast<ColumnInt32 &>(column).getData().push_back(x);
+}
+
+void SerializationDate32::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
+{
+    writeChar('"', ostr);
+    serializeText(column, row_num, ostr, settings);
+    writeChar('"', ostr);
+}
+
+void SerializationDate32::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
+{
+    LocalDate value;
+    readCSV(value, istr);
+    assert_cast<ColumnInt32 &>(column).getData().push_back(value.getExtenedDayNum());
+}
+}

src/DataTypes/Serializations/SerializationDate32.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <DataTypes/Serializations/SerializationNumber.h>
+
+namespace DB
+{
+class SerializationDate32 final : public SerializationNumber<Int32>
+{
+public:
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
+};
+}

src/DataTypes/ya.make

@@ -16,6 +16,7 @@ SRCS(
     DataTypeCustomIPv4AndIPv6.cpp
     DataTypeCustomSimpleAggregateFunction.cpp
     DataTypeDate.cpp
+    DataTypeDate32.cpp
     DataTypeDateTime.cpp
     DataTypeDateTime64.cpp
     DataTypeDecimalBase.cpp
@@ -45,6 +46,7 @@ SRCS(
     Serializations/SerializationArray.cpp
     Serializations/SerializationCustomSimpleText.cpp
     Serializations/SerializationDate.cpp
+    Serializations/SerializationDate32.cpp
     Serializations/SerializationDateTime.cpp
     Serializations/SerializationDateTime64.cpp
     Serializations/SerializationDecimal.cpp

src/Dictionaries/SSDCacheDictionaryStorage.h

@@ -26,8 +26,10 @@
 namespace ProfileEvents
 {
     extern const Event FileOpen;
-    extern const Event WriteBufferAIOWrite;
+    extern const Event AIOWrite;
-    extern const Event WriteBufferAIOWriteBytes;
+    extern const Event AIOWriteBytes;
+    extern const Event AIORead;
+    extern const Event AIOReadBytes;
 }
 
 namespace DB
@@ -531,8 +533,8 @@ public:
 
         auto bytes_written = eventResult(event);
 
-        ProfileEvents::increment(ProfileEvents::WriteBufferAIOWrite);
+        ProfileEvents::increment(ProfileEvents::AIOWrite);
-        ProfileEvents::increment(ProfileEvents::WriteBufferAIOWriteBytes, bytes_written);
+        ProfileEvents::increment(ProfileEvents::AIOWriteBytes, bytes_written);
 
         if (bytes_written != static_cast<decltype(bytes_written)>(block_size * buffer_size_in_blocks))
             throw Exception(ErrorCodes::AIO_WRITE_ERROR,
@@ -600,6 +602,9 @@ public:
                 buffer_size_in_bytes,
                 read_bytes);
 
+        ProfileEvents::increment(ProfileEvents::AIORead);
+        ProfileEvents::increment(ProfileEvents::AIOReadBytes, read_bytes);
+
         SSDCacheBlock block(block_size);
 
         for (size_t i = 0; i < blocks_length; ++i)
@@ -687,6 +692,9 @@ public:
                     throw Exception(ErrorCodes::AIO_READ_ERROR,
                         "GC: AIO failed to read file ({}). Expected bytes ({}). Actual bytes ({})", file_path, block_size, read_bytes);
 
+                ProfileEvents::increment(ProfileEvents::AIORead);
+                ProfileEvents::increment(ProfileEvents::AIOReadBytes, read_bytes);
+
                 char * request_buffer = getRequestBuffer(request);
 
                 // Unpoison the memory returned from an uninstrumented system function.

src/Disks/DiskCacheWrapper.cpp

@@ -90,17 +90,17 @@ DiskCacheWrapper::readFile(
     const String & path,
     size_t buf_size,
     size_t estimated_size,
-    size_t aio_threshold,
+    size_t direct_io_threshold,
     size_t mmap_threshold,
     MMappedFileCache * mmap_cache) const
 {
     if (!cache_file_predicate(path))
-        return DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
+        return DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
 
     LOG_DEBUG(log, "Read file {} from cache", backQuote(path));
 
     if (cache_disk->exists(path))
-        return cache_disk->readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
+        return cache_disk->readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
 
     auto metadata = acquireDownloadMetadata(path);
 
@@ -134,7 +134,7 @@ DiskCacheWrapper::readFile(
 
                 auto tmp_path = path + ".tmp";
                 {
-                    auto src_buffer = DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
+                    auto src_buffer = DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
                     auto dst_buffer = cache_disk->writeFile(tmp_path, buf_size, WriteMode::Rewrite);
                     copyData(*src_buffer, *dst_buffer);
                 }
@@ -158,9 +158,9 @@ DiskCacheWrapper::readFile(
     }
 
     if (metadata->status == DOWNLOADED)
-        return cache_disk->readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
+        return cache_disk->readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
 
-    return DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
+    return DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
 }
 
 std::unique_ptr<WriteBufferFromFileBase>

src/Disks/DiskCacheWrapper.h

@@ -38,7 +38,7 @@ public:
         const String & path,
         size_t buf_size,
         size_t estimated_size,
-        size_t aio_threshold,
+        size_t direct_io_threshold,
         size_t mmap_threshold,
         MMappedFileCache * mmap_cache) const override;
 

src/Disks/DiskDecorator.cpp

@@ -115,9 +115,9 @@ void DiskDecorator::listFiles(const String & path, std::vector<String> & file_na
 
 std::unique_ptr<ReadBufferFromFileBase>
 DiskDecorator::readFile(
-    const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const
+    const String & path, size_t buf_size, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const
 {
-    return delegate->readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
+    return delegate->readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
 }
 
 std::unique_ptr<WriteBufferFromFileBase>

src/Disks/DiskDecorator.h

@@ -39,7 +39,7 @@ public:
         const String & path,
         size_t buf_size,
         size_t estimated_size,
-        size_t aio_threshold,
+        size_t direct_io_threshold,
         size_t mmap_threshold,
         MMappedFileCache * mmap_cache) const override;
 

src/Disks/DiskLocal.cpp

@@ -211,9 +211,9 @@ void DiskLocal::replaceFile(const String & from_path, const String & to_path)
 
 std::unique_ptr<ReadBufferFromFileBase>
 DiskLocal::readFile(
-    const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const
+    const String & path, size_t buf_size, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const
 {
-    return createReadBufferFromFileBase(fs::path(disk_path) / path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size);
+    return createReadBufferFromFileBase(fs::path(disk_path) / path, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache, buf_size);
 }
 
 std::unique_ptr<WriteBufferFromFileBase>

src/Disks/DiskLocal.h

@@ -74,7 +74,7 @@ public:
         const String & path,
         size_t buf_size,
         size_t estimated_size,
-        size_t aio_threshold,
+        size_t direct_io_threshold,
         size_t mmap_threshold,
         MMappedFileCache * mmap_cache) const override;
 

src/Disks/DiskMemory.h

@@ -66,7 +66,7 @@ public:
         const String & path,
         size_t buf_size,
         size_t estimated_size,
-        size_t aio_threshold,
+        size_t direct_io_threshold,
         size_t mmap_threshold,
         MMappedFileCache * mmap_cache) const override;
 

src/Disks/DiskRestartProxy.cpp

@@ -187,11 +187,11 @@ void DiskRestartProxy::listFiles(const String & path, std::vector<String> & file
 }
 
 std::unique_ptr<ReadBufferFromFileBase> DiskRestartProxy::readFile(
-    const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache)
+    const String & path, size_t buf_size, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache)
     const
 {
     ReadLock lock (mutex);
-    auto impl = DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
+    auto impl = DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
     return std::make_unique<RestartAwareReadBuffer>(*this, std::move(impl));
 }
 

src/Disks/DiskRestartProxy.h

@@ -47,7 +47,7 @@ public:
         const String & path,
         size_t buf_size,
         size_t estimated_size,
-        size_t aio_threshold,
+        size_t direct_io_threshold,
         size_t mmap_threshold,
         MMappedFileCache * mmap_cache) const override;
     std::unique_ptr<WriteBufferFromFileBase> writeFile(const String & path, size_t buf_size, WriteMode mode) override;

src/Disks/HDFS/DiskHDFS.h

@@ -48,7 +48,7 @@ public:
         const String & path,
         size_t buf_size,
         size_t estimated_size,
-        size_t aio_threshold,
+        size_t direct_io_threshold,
         size_t mmap_threshold,
         MMappedFileCache * mmap_cache) const override;
 

src/Disks/IDisk.h

@@ -156,7 +156,7 @@ public:
         const String & path,
         size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
         size_t estimated_size = 0,
-        size_t aio_threshold = 0,
+        size_t direct_io_threshold = 0,
         size_t mmap_threshold = 0,
         MMappedFileCache * mmap_cache = nullptr) const = 0;
 

src/Disks/IDiskRemote.cpp

@@ -417,7 +417,11 @@ void IDiskRemote::removeDirectory(const String & path)
 
 DiskDirectoryIteratorPtr IDiskRemote::iterateDirectory(const String & path)
 {
-    return std::make_unique<RemoteDiskDirectoryIterator>(metadata_path + path, path);
+    fs::path meta_path = fs::path(metadata_path) / path;
+    if (fs::exists(meta_path) && fs::is_directory(meta_path))
+        return std::make_unique<RemoteDiskDirectoryIterator>(meta_path, path);
+    else
+        return std::make_unique<RemoteDiskDirectoryIterator>();
 }
 
 

src/Disks/IDiskRemote.h

@@ -193,6 +193,7 @@ struct IDiskRemote::Metadata
 class RemoteDiskDirectoryIterator final : public IDiskDirectoryIterator
 {
 public:
+    RemoteDiskDirectoryIterator() {}
     RemoteDiskDirectoryIterator(const String & full_path, const String & folder_path_) : iter(full_path), folder_path(folder_path_) {}
 
     void next() override { ++iter; }

src/Disks/S3/DiskS3.h

@@ -77,7 +77,7 @@ public:
         const String & path,
         size_t buf_size,
         size_t estimated_size,
-        size_t aio_threshold,
+        size_t direct_io_threshold,
         size_t mmap_threshold,
         MMappedFileCache * mmap_cache) const override;
 

src/Functions/CustomWeekTransforms.h

@@ -42,6 +42,11 @@ struct ToYearWeekImpl
         YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode | static_cast<UInt32>(WeekModeFlag::YEAR));
         return yw.first * 100 + yw.second;
     }
+    static inline UInt32 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)
+    {
+        YearWeek yw = time_zone.toYearWeek(ExtendedDayNum (d), week_mode | static_cast<UInt32>(WeekModeFlag::YEAR));
+        return yw.first * 100 + yw.second;
+    }
     static inline UInt32 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
         YearWeek yw = time_zone.toYearWeek(DayNum(d), week_mode | static_cast<UInt32>(WeekModeFlag::YEAR));
@@ -65,6 +70,10 @@ struct ToStartOfWeekImpl
         return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
 //        return time_zone.toFirstDayNumOfWeek(t, week_mode);
     }
+    static inline UInt16 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode);
+    }
     static inline UInt16 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
         return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode);
@@ -88,6 +97,11 @@ struct ToWeekImpl
         YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode);
         return yw.second;
     }
+    static inline UInt8 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)
+    {
+        YearWeek yw = time_zone.toYearWeek(ExtendedDayNum(d), week_mode);
+        return yw.second;
+    }
     static inline UInt8 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
         YearWeek yw = time_zone.toYearWeek(DayNum(d), week_mode);

src/Functions/DateTimeTransforms.h

@@ -46,6 +46,7 @@ struct ZeroTransform
 {
     static inline UInt16 execute(Int64, const DateLUTImpl &) { return 0; }
     static inline UInt16 execute(UInt32, const DateLUTImpl &) { return 0; }
+    static inline UInt16 execute(Int32, const DateLUTImpl &) { return 0; }
     static inline UInt16 execute(UInt16, const DateLUTImpl &) { return 0; }
 };
 
@@ -61,6 +62,10 @@ struct ToDateImpl
     {
         return UInt16(time_zone.toDayNum(t));
     }
+    static inline UInt16 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl &)
     {
         return d;
@@ -82,6 +87,10 @@ struct ToStartOfDayImpl
     {
         return time_zone.toDate(t);
     }
+    static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toDate(ExtendedDayNum(d));
+    }
     static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toDate(ExtendedDayNum(d));
@@ -104,6 +113,10 @@ struct ToMondayImpl
         //return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t));
         return time_zone.toFirstDayNumOfWeek(t);
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d));
@@ -124,6 +137,10 @@ struct ToStartOfMonthImpl
     {
         return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t));
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d));
@@ -144,6 +161,10 @@ struct ToStartOfQuarterImpl
     {
         return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t));
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d));
@@ -164,6 +185,10 @@ struct ToStartOfYearImpl
     {
         return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t));
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toFirstDayNumOfYear(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toFirstDayNumOfYear(ExtendedDayNum(d));
@@ -186,7 +211,10 @@ struct ToTimeImpl
     {
         return time_zone.toTime(t) + 86400;
     }
-
+    static inline UInt32 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
     static inline UInt32 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -207,6 +235,10 @@ struct ToStartOfMinuteImpl
     {
         return time_zone.toStartOfMinute(t);
     }
+    static inline UInt32 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
     static inline UInt32 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -242,6 +274,10 @@ struct ToStartOfSecondImpl
     {
         throw Exception("Illegal type DateTime of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
     }
+    static inline UInt32 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
     static inline UInt32 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -262,6 +298,10 @@ struct ToStartOfFiveMinuteImpl
     {
         return time_zone.toStartOfFiveMinute(t);
     }
+    static inline UInt32 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
     static inline UInt32 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -282,6 +322,10 @@ struct ToStartOfTenMinutesImpl
     {
         return time_zone.toStartOfTenMinutes(t);
     }
+    static inline UInt32 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
     static inline UInt32 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -302,6 +346,10 @@ struct ToStartOfFifteenMinutesImpl
     {
         return time_zone.toStartOfFifteenMinutes(t);
     }
+    static inline UInt32 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
     static inline UInt32 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -326,6 +374,11 @@ struct TimeSlotImpl
         return t / 1800 * 1800;
     }
 
+    static inline UInt32 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
+
     static inline UInt32 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -348,6 +401,11 @@ struct ToStartOfHourImpl
         return time_zone.toStartOfHour(t);
     }
 
+    static inline UInt32 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
+
     static inline UInt32 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -368,6 +426,10 @@ struct ToYearImpl
     {
         return time_zone.toYear(t);
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toYear(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toYear(ExtendedDayNum(d));
@@ -388,6 +450,10 @@ struct ToQuarterImpl
     {
         return time_zone.toQuarter(t);
     }
+    static inline UInt8 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toQuarter(ExtendedDayNum(d));
+    }
     static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toQuarter(ExtendedDayNum(d));
@@ -408,6 +474,10 @@ struct ToMonthImpl
     {
         return time_zone.toMonth(t);
     }
+    static inline UInt8 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toMonth(ExtendedDayNum(d));
+    }
     static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toMonth(ExtendedDayNum(d));
@@ -428,6 +498,10 @@ struct ToDayOfMonthImpl
     {
         return time_zone.toDayOfMonth(t);
     }
+    static inline UInt8 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toDayOfMonth(ExtendedDayNum(d));
+    }
     static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toDayOfMonth(ExtendedDayNum(d));
@@ -448,6 +522,10 @@ struct ToDayOfWeekImpl
     {
         return time_zone.toDayOfWeek(t);
     }
+    static inline UInt8 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toDayOfWeek(ExtendedDayNum(d));
+    }
     static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toDayOfWeek(ExtendedDayNum(d));
@@ -468,6 +546,10 @@ struct ToDayOfYearImpl
     {
         return time_zone.toDayOfYear(t);
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toDayOfYear(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toDayOfYear(ExtendedDayNum(d));
@@ -488,7 +570,10 @@ struct ToHourImpl
     {
         return time_zone.toHour(t);
     }
-
+    static inline UInt8 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
     static inline UInt8 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -511,6 +596,11 @@ struct TimezoneOffsetImpl
         return time_zone.timezoneOffset(t);
     }
 
+    static inline time_t execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
+
     static inline time_t execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -531,6 +621,10 @@ struct ToMinuteImpl
     {
         return time_zone.toMinute(t);
     }
+    static inline UInt8 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
     static inline UInt8 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -551,6 +645,10 @@ struct ToSecondImpl
     {
         return time_zone.toSecond(t);
     }
+    static inline UInt8 execute(Int32, const DateLUTImpl &)
+    {
+        return dateIsNotSupported(name);
+    }
     static inline UInt8 execute(UInt16, const DateLUTImpl &)
     {
         return dateIsNotSupported(name);
@@ -571,6 +669,10 @@ struct ToISOYearImpl
     {
         return time_zone.toISOYear(time_zone.toDayNum(t));
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toISOYear(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toISOYear(ExtendedDayNum(d));
@@ -591,6 +693,10 @@ struct ToStartOfISOYearImpl
     {
         return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t));
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d));
@@ -611,6 +717,10 @@ struct ToISOWeekImpl
     {
         return time_zone.toISOWeek(time_zone.toDayNum(t));
     }
+    static inline UInt8 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toISOWeek(ExtendedDayNum(d));
+    }
     static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toISOWeek(ExtendedDayNum(d));
@@ -631,6 +741,10 @@ struct ToRelativeYearNumImpl
     {
         return time_zone.toYear(static_cast<time_t>(t));
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toYear(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toYear(ExtendedDayNum(d));
@@ -651,6 +765,10 @@ struct ToRelativeQuarterNumImpl
     {
         return time_zone.toRelativeQuarterNum(static_cast<time_t>(t));
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toRelativeQuarterNum(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toRelativeQuarterNum(ExtendedDayNum(d));
@@ -671,6 +789,10 @@ struct ToRelativeMonthNumImpl
     {
         return time_zone.toRelativeMonthNum(static_cast<time_t>(t));
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toRelativeMonthNum(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toRelativeMonthNum(ExtendedDayNum(d));
@@ -691,6 +813,10 @@ struct ToRelativeWeekNumImpl
     {
         return time_zone.toRelativeWeekNum(static_cast<time_t>(t));
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toRelativeWeekNum(ExtendedDayNum(d));
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toRelativeWeekNum(ExtendedDayNum(d));
@@ -711,6 +837,10 @@ struct ToRelativeDayNumImpl
     {
         return time_zone.toDayNum(static_cast<time_t>(t));
     }
+    static inline UInt16 execute(Int32 d, const DateLUTImpl &)
+    {
+        return static_cast<ExtendedDayNum>(d);
+    }
     static inline UInt16 execute(UInt16 d, const DateLUTImpl &)
     {
         return static_cast<DayNum>(d);
@@ -732,6 +862,10 @@ struct ToRelativeHourNumImpl
     {
         return time_zone.toRelativeHourNum(static_cast<time_t>(t));
     }
+    static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toRelativeHourNum(ExtendedDayNum(d));
+    }
     static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toRelativeHourNum(ExtendedDayNum(d));
@@ -752,6 +886,10 @@ struct ToRelativeMinuteNumImpl
     {
         return time_zone.toRelativeMinuteNum(static_cast<time_t>(t));
     }
+    static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toRelativeMinuteNum(ExtendedDayNum(d));
+    }
     static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toRelativeMinuteNum(ExtendedDayNum(d));
@@ -772,6 +910,10 @@ struct ToRelativeSecondNumImpl
     {
         return t;
     }
+    static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.fromDayNum(ExtendedDayNum(d));
+    }
     static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.fromDayNum(ExtendedDayNum(d));
@@ -792,6 +934,10 @@ struct ToYYYYMMImpl
     {
         return time_zone.toNumYYYYMM(t);
     }
+    static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toNumYYYYMM(static_cast<ExtendedDayNum>(d));
+    }
     static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toNumYYYYMM(static_cast<DayNum>(d));
@@ -812,6 +958,10 @@ struct ToYYYYMMDDImpl
     {
         return time_zone.toNumYYYYMMDD(t);
     }
+    static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toNumYYYYMMDD(static_cast<ExtendedDayNum>(d));
+    }
     static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toNumYYYYMMDD(static_cast<DayNum>(d));
@@ -832,6 +982,10 @@ struct ToYYYYMMDDhhmmssImpl
     {
         return time_zone.toNumYYYYMMDDhhmmss(t);
     }
+    static inline UInt64 execute(Int32 d, const DateLUTImpl & time_zone)
+    {
+        return time_zone.toNumYYYYMMDDhhmmss(time_zone.toDate(static_cast<ExtendedDayNum>(d)));
+    }
     static inline UInt64 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
         return time_zone.toNumYYYYMMDDhhmmss(time_zone.toDate(static_cast<DayNum>(d)));

src/Functions/FunctionCustomWeekToSomething.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeDateTime64.h>
 #include <Functions/CustomWeekTransforms.h>
@@ -35,7 +36,7 @@ public:
     {
         if (arguments.size() == 1)
         {
-            if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
+            if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
                 throw Exception(
                     "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName()
                         + ". Should be a date or a date with time",
@@ -43,7 +44,7 @@ public:
         }
         else if (arguments.size() == 2)
         {
-            if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
+            if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
                 throw Exception(
                     "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName()
                         + ". Should be a date or a date with time",
@@ -59,7 +60,7 @@ public:
         }
         else if (arguments.size() == 3)
         {
-            if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
+            if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
                 throw Exception(
                     "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName()
                         + ". Should be a date or a date with time",
@@ -105,6 +106,9 @@ public:
         if (which.isDate())
             return CustomWeekTransformImpl<DataTypeDate, ToDataType>::execute(
                 arguments, result_type, input_rows_count, Transform{});
+        else if (which.isDate32())
+            return CustomWeekTransformImpl<DataTypeDate32, ToDataType>::execute(
+                arguments, result_type, input_rows_count, Transform{});
         else if (which.isDateTime())
             return CustomWeekTransformImpl<DataTypeDateTime, ToDataType>::execute(
                 arguments, result_type, input_rows_count, Transform{});

src/Functions/FunctionDateOrDateTimeAddInterval.h

@@ -2,6 +2,7 @@
 #include <common/DateLUTImpl.h>
 
 #include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeDateTime64.h>
 
@@ -50,7 +51,11 @@ struct AddSecondsImpl
     {
         return t + delta;
     }
-
+    static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
+    {
+        // use default datetime64 scale
+        return (time_zone.fromDayNum(ExtendedDayNum(d)) + delta) * 1000;
+    }
     static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
     {
         return time_zone.fromDayNum(ExtendedDayNum(d)) + delta;
@@ -71,7 +76,11 @@ struct AddMinutesImpl
     {
         return t + delta * 60;
     }
-
+    static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
+    {
+        // use default datetime64 scale
+        return (time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 60) * 1000;
+    }
     static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
     {
         return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 60;
@@ -91,7 +100,11 @@ struct AddHoursImpl
     {
         return t + delta * 3600;
     }
-
+    static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
+    {
+        // use default datetime64 scale
+        return (time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 3600) * 1000;
+    }
     static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
     {
         return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 3600;
@@ -117,6 +130,11 @@ struct AddDaysImpl
     {
         return d + delta;
     }
+
+    static inline NO_SANITIZE_UNDEFINED Int32 execute(Int32 d, Int64 delta, const DateLUTImpl &)
+    {
+        return d + delta;
+    }
 };
 
 struct AddWeeksImpl
@@ -124,17 +142,22 @@ struct AddWeeksImpl
     static constexpr auto name = "addWeeks";
 
     static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents<DateTime64>
-    execute(DecimalUtils::DecimalComponents<DateTime64> t, Int64 delta, const DateLUTImpl & time_zone)
+    execute(DecimalUtils::DecimalComponents<DateTime64> t, Int32 delta, const DateLUTImpl & time_zone)
     {
         return {time_zone.addWeeks(t.whole, delta), t.fractional};
     }
 
-    static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone)
+    static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int32 delta, const DateLUTImpl & time_zone)
     {
         return time_zone.addWeeks(t, delta);
     }
 
-    static inline NO_SANITIZE_UNDEFINED UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl &)
+    static inline NO_SANITIZE_UNDEFINED UInt16 execute(UInt16 d, Int32 delta, const DateLUTImpl &)
+    {
+        return d + delta * 7;
+    }
+
+    static inline NO_SANITIZE_UNDEFINED Int32 execute(Int32 d, Int32 delta, const DateLUTImpl &)
     {
         return d + delta * 7;
     }
@@ -159,6 +182,11 @@ struct AddMonthsImpl
     {
         return time_zone.addMonths(ExtendedDayNum(d), delta);
     }
+
+    static inline Int32 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
+    {
+        return time_zone.addMonths(ExtendedDayNum(d), delta);
+    }
 };
 
 struct AddQuartersImpl
@@ -166,17 +194,22 @@ struct AddQuartersImpl
     static constexpr auto name = "addQuarters";
 
     static inline DecimalUtils::DecimalComponents<DateTime64>
-    execute(DecimalUtils::DecimalComponents<DateTime64> t, Int64 delta, const DateLUTImpl & time_zone)
+    execute(DecimalUtils::DecimalComponents<DateTime64> t, Int32 delta, const DateLUTImpl & time_zone)
     {
         return {time_zone.addQuarters(t.whole, delta), t.fractional};
     }
 
-    static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone)
+    static inline UInt32 execute(UInt32 t, Int32 delta, const DateLUTImpl & time_zone)
     {
         return time_zone.addQuarters(t, delta);
     }
 
-    static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
+    static inline UInt16 execute(UInt16 d, Int32 delta, const DateLUTImpl & time_zone)
+    {
+        return time_zone.addQuarters(ExtendedDayNum(d), delta);
+    }
+
+    static inline Int32 execute(Int32 d, Int32 delta, const DateLUTImpl & time_zone)
     {
         return time_zone.addQuarters(ExtendedDayNum(d), delta);
     }
@@ -201,6 +234,11 @@ struct AddYearsImpl
     {
         return time_zone.addYears(ExtendedDayNum(d), delta);
     }
+
+    static inline Int32 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
+    {
+        return time_zone.addYears(ExtendedDayNum(d), delta);
+    }
 };
 
 template <typename Transform>
@@ -342,7 +380,7 @@ template <typename FieldType> struct ResultDataTypeMap {};
 template <> struct ResultDataTypeMap<UInt16>     { using ResultDataType = DataTypeDate; };
 template <> struct ResultDataTypeMap<Int16>      { using ResultDataType = DataTypeDate; };
 template <> struct ResultDataTypeMap<UInt32>     { using ResultDataType = DataTypeDateTime; };
-template <> struct ResultDataTypeMap<Int32>      { using ResultDataType = DataTypeDateTime; };
+template <> struct ResultDataTypeMap<Int32>      { using ResultDataType = DataTypeDate32; };
 template <> struct ResultDataTypeMap<DateTime64> { using ResultDataType = DataTypeDateTime64; };
 template <> struct ResultDataTypeMap<Int64>      { using ResultDataType = DataTypeDateTime64; };
 }
@@ -375,7 +413,7 @@ public:
 
         if (arguments.size() == 2)
         {
-            if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
+            if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
                 throw Exception{"Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() +
                     ". Should be a date or a date with time", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
         }
@@ -398,6 +436,8 @@ public:
         {
             case TypeIndex::Date:
                 return resolveReturnType<DataTypeDate>(arguments);
+            case TypeIndex::Date32:
+                return resolveReturnType<DataTypeDate32>(arguments);
             case TypeIndex::DateTime:
                 return resolveReturnType<DataTypeDateTime>(arguments);
             case TypeIndex::DateTime64:
@@ -437,16 +477,23 @@ public:
 
         if constexpr (std::is_same_v<ResultDataType, DataTypeDate>)
             return std::make_shared<DataTypeDate>();
+        else if constexpr (std::is_same_v<ResultDataType, DataTypeDate32>)
+            return std::make_shared<DataTypeDate32>();
         else if constexpr (std::is_same_v<ResultDataType, DataTypeDateTime>)
         {
             return std::make_shared<DataTypeDateTime>(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
         }
         else if constexpr (std::is_same_v<ResultDataType, DataTypeDateTime64>)
         {
-            // TODO (vnemkov): what if there is an overload of Transform::execute() that returns DateTime64 from DateTime or Date ?
+            if (typeid_cast<const DataTypeDateTime64 *>(arguments[0].type.get()))
-            // Shall we use the default scale or one from optional argument ?
+            {
-            const auto & datetime64_type = assert_cast<const DataTypeDateTime64 &>(*arguments[0].type);
+                const auto & datetime64_type = assert_cast<const DataTypeDateTime64 &>(*arguments[0].type);
-            return std::make_shared<DataTypeDateTime64>(datetime64_type.getScale(), extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
+                return std::make_shared<DataTypeDateTime64>(datetime64_type.getScale(), extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
+            }
+            else
+            {
+                return std::make_shared<DataTypeDateTime64>(DataTypeDateTime64::default_scale, extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
+            }
         }
         else
         {
@@ -470,6 +517,11 @@ public:
             return DateTimeAddIntervalImpl<DataTypeDate, TransformResultDataType<DataTypeDate>, Transform>::execute(
                 Transform{}, arguments, result_type);
         }
+        else if (which.isDate32())
+        {
+            return DateTimeAddIntervalImpl<DataTypeDate32, TransformResultDataType<DataTypeDate32>, Transform>::execute(
+                Transform{}, arguments, result_type);
+        }
         else if (which.isDateTime())
         {
             return DateTimeAddIntervalImpl<DataTypeDateTime, TransformResultDataType<DataTypeDateTime>, Transform>::execute(

src/Functions/FunctionDateOrDateTimeToSomething.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <Functions/IFunction.h>
 #include <DataTypes/DataTypeDateTime64.h>
@@ -38,7 +39,7 @@ public:
     {
         if (arguments.size() == 1)
         {
-            if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
+            if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
                 throw Exception(
                     "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName()
                         + ". Should be a date or a date with time",
@@ -46,7 +47,7 @@ public:
         }
         else if (arguments.size() == 2)
         {
-            if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
+            if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
                 throw Exception(
                     "Illegal type " + arguments[0].type->getName() + " of argument of function " + getName()
                         + ". Should be a date or a date with time",
@@ -57,7 +58,7 @@ public:
                           "must be of type Date or DateTime. The 2nd argument (optional) must be "
                           "a constant string with timezone name",
                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
-            if (isDate(arguments[0].type) && std::is_same_v<ToDataType, DataTypeDate>)
+            if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) && (std::is_same_v<ToDataType, DataTypeDate> || std::is_same_v<ToDataType, DataTypeDate32>))
                 throw Exception(
                     "The timezone argument of function " + getName() + " is allowed only when the 1st argument has the type DateTime",
                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@@ -103,6 +104,8 @@ public:
 
         if (which.isDate())
             return DateTimeTransformImpl<DataTypeDate, ToDataType, Transform>::execute(arguments, result_type, input_rows_count);
+        else if (which.isDate32())
+            return DateTimeTransformImpl<DataTypeDate32, ToDataType, Transform>::execute(arguments, result_type, input_rows_count);
         else if (which.isDateTime())
             return DateTimeTransformImpl<DataTypeDateTime, ToDataType, Transform>::execute(arguments, result_type, input_rows_count);
         else if (which.isDateTime64())
@@ -146,6 +149,12 @@ public:
                 == Transform::FactorTransform::execute(UInt16(right.get<UInt64>()), date_lut)
                 ? is_monotonic : is_not_monotonic;
         }
+        else if (checkAndGetDataType<DataTypeDate32>(&type))
+        {
+            return Transform::FactorTransform::execute(Int32(left.get<UInt64>()), date_lut)
+                   == Transform::FactorTransform::execute(Int32(right.get<UInt64>()), date_lut)
+                   ? is_monotonic : is_not_monotonic;
+        }
         else
         {
             return Transform::FactorTransform::execute(UInt32(left.get<UInt64>()), date_lut)

src/Functions/FunctionJoinGet.h

@@ -28,7 +28,7 @@ public:
     static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet";
 
     bool useDefaultImplementationForNulls() const override { return false; }
-    bool useDefaultImplementationForLowCardinalityColumns() const override { return true; }
+    bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
     bool useDefaultImplementationForConstants() const override { return true; }
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override;

src/Functions/FunctionsCoding.h

@@ -19,6 +19,7 @@
 #include <Functions/FunctionHelpers.h>
 #include <Functions/IFunction.h>
 #include <Interpreters/Context_fwd.h>
+#include <Interpreters/castColumn.h>
 #include <IO/WriteHelpers.h>
 #include <Common/IPv6ToBinary.h>
 #include <Common/formatIPv6.h>
@@ -978,7 +979,8 @@ public:
             !which.isDateTime64() &&
             !which.isUInt() &&
             !which.isFloat() &&
-            !which.isDecimal())
+            !which.isDecimal() &&
+            !which.isAggregateFunction())
             throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
                 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 
@@ -990,6 +992,15 @@ public:
         const IColumn * column = arguments[0].column.get();
         ColumnPtr res_column;
 
+        WhichDataType which(column->getDataType());
+        if (which.isAggregateFunction())
+        {
+            const ColumnPtr to_string = castColumn(arguments[0], std::make_shared<DataTypeString>());
+            const auto * str_column = checkAndGetColumn<ColumnString>(to_string.get());
+            tryExecuteString(str_column, res_column);
+            return res_column;
+        }
+
         if (tryExecuteUInt<UInt8>(column, res_column) ||
             tryExecuteUInt<UInt16>(column, res_column) ||
             tryExecuteUInt<UInt32>(column, res_column) ||

src/Functions/FunctionsComparison.h

@@ -1081,7 +1081,7 @@ public:
         const DataTypeTuple * right_tuple = checkAndGetDataType<DataTypeTuple>(arguments[1].get());
 
         bool both_represented_by_number = arguments[0]->isValueRepresentedByNumber() && arguments[1]->isValueRepresentedByNumber();
-        bool has_date = left.isDate() || right.isDate();
+        bool has_date = left.isDateOrDate32() || right.isDateOrDate32();
 
         if (!((both_represented_by_number && !has_date)   /// Do not allow to compare date and number.
             || (left.isStringOrFixedString() || right.isStringOrFixedString())  /// Everything can be compared with string by conversion.

src/Functions/FunctionsConversion.cpp

@@ -32,7 +32,7 @@ void registerFunctionsConversion(FunctionFactory & factory)
     factory.registerFunction<FunctionToDate>();
     /// MysQL compatibility alias.
     factory.registerFunction<FunctionToDate>("DATE", FunctionFactory::CaseInsensitive);
-
+    factory.registerFunction<FunctionToDate32>();
     factory.registerFunction<FunctionToDateTime>();
     factory.registerFunction<FunctionToDateTime32>();
     factory.registerFunction<FunctionToDateTime64>();
@@ -62,6 +62,7 @@ void registerFunctionsConversion(FunctionFactory & factory)
     factory.registerFunction<FunctionToFloat32OrZero>();
     factory.registerFunction<FunctionToFloat64OrZero>();
     factory.registerFunction<FunctionToDateOrZero>();
+    factory.registerFunction<FunctionToDate32OrZero>();
     factory.registerFunction<FunctionToDateTimeOrZero>();
     factory.registerFunction<FunctionToDateTime64OrZero>();
 
@@ -87,6 +88,7 @@ void registerFunctionsConversion(FunctionFactory & factory)
     factory.registerFunction<FunctionToFloat32OrNull>();
     factory.registerFunction<FunctionToFloat64OrNull>();
     factory.registerFunction<FunctionToDateOrNull>();
+    factory.registerFunction<FunctionToDate32OrNull>();
     factory.registerFunction<FunctionToDateTimeOrNull>();
     factory.registerFunction<FunctionToDateTime64OrNull>();
 

src/Functions/FunctionsConversion.h

@@ -12,6 +12,7 @@
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeFixedString.h>
 #include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeDateTime64.h>
 #include <DataTypes/DataTypeEnum.h>
@@ -670,6 +671,8 @@ struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType,
 
             if constexpr (std::is_same_v<FromDataType, DataTypeDate>)
                 data_to.resize(size * (strlen("YYYY-MM-DD") + 1));
+            else if constexpr (std::is_same_v<FromDataType, DataTypeDate32>)
+                data_to.resize(size * (strlen("YYYY-MM-DD") + 1));
             else if constexpr (std::is_same_v<FromDataType, DataTypeDateTime>)
                 data_to.resize(size * (strlen("YYYY-MM-DD hh:mm:ss") + 1));
             else if constexpr (std::is_same_v<FromDataType, DataTypeDateTime64>)
@@ -751,6 +754,14 @@ inline void parseImpl<DataTypeDate>(DataTypeDate::FieldType & x, ReadBuffer & rb
     x = tmp;
 }
 
+template <>
+inline void parseImpl<DataTypeDate32>(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl *)
+{
+    ExtendedDayNum tmp(0);
+    readDateText(tmp, rb);
+    x = tmp;
+}
+
 // NOTE: no need of extra overload of DateTime64, since readDateTimeText64 has different signature and that case is explicitly handled in the calling code.
 template <>
 inline void parseImpl<DataTypeDateTime>(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
@@ -791,6 +802,16 @@ inline bool tryParseImpl<DataTypeDate>(DataTypeDate::FieldType & x, ReadBuffer &
     return true;
 }
 
+template <>
+inline bool tryParseImpl<DataTypeDate32>(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl *)
+{
+    ExtendedDayNum tmp(0);
+    if (!tryReadDateText(tmp, rb))
+        return false;
+    x = tmp;
+    return true;
+}
+
 template <>
 inline bool tryParseImpl<DataTypeDateTime>(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
 {
@@ -1215,6 +1236,7 @@ struct ConvertImpl<DataTypeFixedString, DataTypeString, Name, ConvertDefaultBeha
 
 /// Declared early because used below.
 struct NameToDate { static constexpr auto name = "toDate"; };
+struct NameToDate32 { static constexpr auto name = "toDate32"; };
 struct NameToDateTime { static constexpr auto name = "toDateTime"; };
 struct NameToDateTime32 { static constexpr auto name = "toDateTime32"; };
 struct NameToDateTime64 { static constexpr auto name = "toDateTime64"; };
@@ -1890,7 +1912,7 @@ struct ToDateMonotonicity
     static IFunction::Monotonicity get(const IDataType & type, const Field & left, const Field & right)
     {
         auto which = WhichDataType(type);
-        if (which.isDate() || which.isDateTime() || which.isDateTime64() || which.isInt8() || which.isInt16() || which.isUInt8() || which.isUInt16())
+        if (which.isDate() || which.isDate32() || which.isDateTime() || which.isDateTime64() || which.isInt8() || which.isInt16() || which.isUInt8() || which.isUInt16())
             return {true, true, true};
         else if (
             (which.isUInt() && ((left.isNull() || left.get<UInt64>() < 0xFFFF) && (right.isNull() || right.get<UInt64>() >= 0xFFFF)))
@@ -1991,6 +2013,7 @@ using FunctionToInt256 = FunctionConvert<DataTypeInt256, NameToInt256, ToNumberM
 using FunctionToFloat32 = FunctionConvert<DataTypeFloat32, NameToFloat32, ToNumberMonotonicity<Float32>>;
 using FunctionToFloat64 = FunctionConvert<DataTypeFloat64, NameToFloat64, ToNumberMonotonicity<Float64>>;
 using FunctionToDate = FunctionConvert<DataTypeDate, NameToDate, ToDateMonotonicity>;
+using FunctionToDate32 = FunctionConvert<DataTypeDate32, NameToDate32, ToDateMonotonicity>;
 using FunctionToDateTime = FunctionConvert<DataTypeDateTime, NameToDateTime, ToDateTimeMonotonicity>;
 using FunctionToDateTime32 = FunctionConvert<DataTypeDateTime, NameToDateTime32, ToDateTimeMonotonicity>;
 using FunctionToDateTime64 = FunctionConvert<DataTypeDateTime64, NameToDateTime64, UnknownMonotonicity>;
@@ -2050,6 +2073,7 @@ struct NameToInt256OrZero { static constexpr auto name = "toInt256OrZero"; };
 struct NameToFloat32OrZero { static constexpr auto name = "toFloat32OrZero"; };
 struct NameToFloat64OrZero { static constexpr auto name = "toFloat64OrZero"; };
 struct NameToDateOrZero { static constexpr auto name = "toDateOrZero"; };
+struct NameToDate32OrZero { static constexpr auto name = "toDate32OrZero"; };
 struct NameToDateTimeOrZero { static constexpr auto name = "toDateTimeOrZero"; };
 struct NameToDateTime64OrZero { static constexpr auto name = "toDateTime64OrZero"; };
 struct NameToDecimal32OrZero { static constexpr auto name = "toDecimal32OrZero"; };
@@ -2073,6 +2097,7 @@ using FunctionToInt256OrZero = FunctionConvertFromString<DataTypeInt256, NameToI
 using FunctionToFloat32OrZero = FunctionConvertFromString<DataTypeFloat32, NameToFloat32OrZero, ConvertFromStringExceptionMode::Zero>;
 using FunctionToFloat64OrZero = FunctionConvertFromString<DataTypeFloat64, NameToFloat64OrZero, ConvertFromStringExceptionMode::Zero>;
 using FunctionToDateOrZero = FunctionConvertFromString<DataTypeDate, NameToDateOrZero, ConvertFromStringExceptionMode::Zero>;
+using FunctionToDate32OrZero = FunctionConvertFromString<DataTypeDate32, NameToDate32OrZero, ConvertFromStringExceptionMode::Zero>;
 using FunctionToDateTimeOrZero = FunctionConvertFromString<DataTypeDateTime, NameToDateTimeOrZero, ConvertFromStringExceptionMode::Zero>;
 using FunctionToDateTime64OrZero = FunctionConvertFromString<DataTypeDateTime64, NameToDateTime64OrZero, ConvertFromStringExceptionMode::Zero>;
 using FunctionToDecimal32OrZero = FunctionConvertFromString<DataTypeDecimal<Decimal32>, NameToDecimal32OrZero, ConvertFromStringExceptionMode::Zero>;
@@ -2096,6 +2121,7 @@ struct NameToInt256OrNull { static constexpr auto name = "toInt256OrNull"; };
 struct NameToFloat32OrNull { static constexpr auto name = "toFloat32OrNull"; };
 struct NameToFloat64OrNull { static constexpr auto name = "toFloat64OrNull"; };
 struct NameToDateOrNull { static constexpr auto name = "toDateOrNull"; };
+struct NameToDate32OrNull { static constexpr auto name = "toDate32OrNull"; };
 struct NameToDateTimeOrNull { static constexpr auto name = "toDateTimeOrNull"; };
 struct NameToDateTime64OrNull { static constexpr auto name = "toDateTime64OrNull"; };
 struct NameToDecimal32OrNull { static constexpr auto name = "toDecimal32OrNull"; };
@@ -2119,6 +2145,7 @@ using FunctionToInt256OrNull = FunctionConvertFromString<DataTypeInt256, NameToI
 using FunctionToFloat32OrNull = FunctionConvertFromString<DataTypeFloat32, NameToFloat32OrNull, ConvertFromStringExceptionMode::Null>;
 using FunctionToFloat64OrNull = FunctionConvertFromString<DataTypeFloat64, NameToFloat64OrNull, ConvertFromStringExceptionMode::Null>;
 using FunctionToDateOrNull = FunctionConvertFromString<DataTypeDate, NameToDateOrNull, ConvertFromStringExceptionMode::Null>;
+using FunctionToDate32OrNull = FunctionConvertFromString<DataTypeDate32, NameToDate32OrNull, ConvertFromStringExceptionMode::Null>;
 using FunctionToDateTimeOrNull = FunctionConvertFromString<DataTypeDateTime, NameToDateTimeOrNull, ConvertFromStringExceptionMode::Null>;
 using FunctionToDateTime64OrNull = FunctionConvertFromString<DataTypeDateTime64, NameToDateTime64OrNull, ConvertFromStringExceptionMode::Null>;
 using FunctionToDecimal32OrNull = FunctionConvertFromString<DataTypeDecimal<Decimal32>, NameToDecimal32OrNull, ConvertFromStringExceptionMode::Null>;

src/Functions/FunctionsExternalDictionaries.h

@@ -163,13 +163,6 @@ public:
                 arguments[0]->getName(),
                 getName());
 
-        if (!WhichDataType(arguments[1]).isUInt64() &&
-            !isTuple(arguments[1]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "Illegal type {} of second argument of function {} must be UInt64 or tuple(...)",
-                arguments[1]->getName(),
-                getName());
-
         return std::make_shared<DataTypeUInt8>();
     }
 
@@ -189,8 +182,8 @@ public:
         auto dictionary_key_type = dictionary->getKeyType();
 
         const ColumnWithTypeAndName & key_column_with_type = arguments[1];
-        const auto key_column = key_column_with_type.column;
+        auto key_column = key_column_with_type.column;
-        const auto key_column_type = WhichDataType(key_column_with_type.type);
+        auto key_column_type = key_column_with_type.type;
 
         ColumnPtr range_col = nullptr;
         DataTypePtr range_col_type = nullptr;
@@ -214,7 +207,7 @@ public:
 
         if (dictionary_key_type == DictionaryKeyType::simple)
         {
-            if (!key_column_type.isUInt64())
+            if (!WhichDataType(key_column_type).isUInt64())
                  throw Exception(
                      ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
                      "Second argument of function {} must be UInt64 when dictionary is simple. Actual type {}.",
@@ -225,24 +218,39 @@ public:
         }
         else if (dictionary_key_type == DictionaryKeyType::complex)
         {
-            if (!key_column_type.isTuple())
-                throw Exception(
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Second argument of function {} must be tuple when dictionary is complex. Actual type {}.",
-                    getName(),
-                    key_column_with_type.type->getName());
-
             /// Functions in external dictionaries_loader only support full-value (not constant) columns with keys.
-            ColumnPtr key_column_full = key_column->convertToFullColumnIfConst();
+            key_column = key_column->convertToFullColumnIfConst();
+            size_t keys_size = dictionary->getStructure().getKeysSize();
 
-            const auto & key_columns = typeid_cast<const ColumnTuple &>(*key_column_full).getColumnsCopy();
+            if (!isTuple(key_column_type))
-            const auto & key_types = static_cast<const DataTypeTuple &>(*key_column_with_type.type).getElements();
+            {
+                if (keys_size > 1)
+                {
+                    throw Exception(
+                        ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                        "Third argument of function {} must be tuple when dictionary is complex and key contains more than 1 attribute."
+                        "Actual type {}.",
+                        getName(),
+                        key_column_type->getName());
+                }
+                else
+                {
+                    Columns tuple_columns = {std::move(key_column)};
+                    key_column = ColumnTuple::create(tuple_columns);
+
+                    DataTypes tuple_types = {key_column_type};
+                    key_column_type = std::make_shared<DataTypeTuple>(tuple_types);
+                }
+            }
+
+            const auto & key_columns = assert_cast<const ColumnTuple &>(*key_column).getColumnsCopy();
+            const auto & key_types = assert_cast<const DataTypeTuple &>(*key_column_type).getElements();
 
             return dictionary->hasKeys(key_columns, key_types);
         }
         else
         {
-            if (!key_column_type.isUInt64())
+            if (!WhichDataType(key_column_type).isUInt64())
                 throw Exception(
                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
                     "Second argument of function {} must be UInt64 when dictionary is range. Actual type {}.",
@@ -346,13 +354,6 @@ public:
         Strings attribute_names = getAttributeNamesFromColumn(arguments[1].column, arguments[1].type);
 
         auto dictionary = helper.getDictionary(dictionary_name);
-
-        if (!WhichDataType(arguments[2].type).isUInt64() && !isTuple(arguments[2].type))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "Illegal type {} of third argument of function {}, must be UInt64 or tuple(...).",
-                arguments[2].type->getName(),
-                getName());
-
         auto dictionary_key_type = dictionary->getKeyType();
 
         size_t current_arguments_index = 3;
@@ -446,18 +447,35 @@ public:
         }
         else if (dictionary_key_type == DictionaryKeyType::complex)
         {
-            if (!isTuple(key_col_with_type.type))
-                 throw Exception(
-                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                     "Third argument of function {} must be tuple when dictionary is complex. Actual type {}.",
-                     getName(),
-                     key_col_with_type.type->getName());
-
             /// Functions in external dictionaries_loader only support full-value (not constant) columns with keys.
-            ColumnPtr key_column_full = key_col_with_type.column->convertToFullColumnIfConst();
+            ColumnPtr key_column = key_col_with_type.column->convertToFullColumnIfConst();
+            DataTypePtr key_column_type = key_col_with_type.type;
 
-            const auto & key_columns = typeid_cast<const ColumnTuple &>(*key_column_full).getColumnsCopy();
+            size_t keys_size = dictionary->getStructure().getKeysSize();
-            const auto & key_types = static_cast<const DataTypeTuple &>(*key_col_with_type.type).getElements();
+
+            if (!isTuple(key_column_type))
+            {
+                if (keys_size > 1)
+                {
+                    throw Exception(
+                         ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                         "Third argument of function {} must be tuple when dictionary is complex and key contains more than 1 attribute."
+                         "Actual type {}.",
+                         getName(),
+                         key_col_with_type.type->getName());
+                }
+                else
+                {
+                    Columns tuple_columns = {std::move(key_column)};
+                    key_column = ColumnTuple::create(tuple_columns);
+
+                    DataTypes tuple_types = {key_column_type};
+                    key_column_type = std::make_shared<DataTypeTuple>(tuple_types);
+                }
+            }
+
+            const auto & key_columns = assert_cast<const ColumnTuple &>(*key_column).getColumnsCopy();
+            const auto & key_types = assert_cast<const DataTypeTuple &>(*key_column_type).getElements();
 
             result = executeDictionaryRequest(
                 dictionary,

src/Functions/FunctionsHashing.h

@@ -683,6 +683,8 @@ public:
             return executeType<Int64>(arguments);
         else if (which.isDate())
             return executeType<UInt16>(arguments);
+        else if (which.isDate32())
+            return executeType<Int32>(arguments);
         else if (which.isDateTime())
             return executeType<UInt32>(arguments);
         else if (which.isDecimal32())
@@ -986,6 +988,7 @@ private:
         else if (which.isEnum8()) executeIntType<Int8, first>(icolumn, vec_to);
         else if (which.isEnum16()) executeIntType<Int16, first>(icolumn, vec_to);
         else if (which.isDate()) executeIntType<UInt16, first>(icolumn, vec_to);
+        else if (which.isDate32()) executeIntType<Int32, first>(icolumn, vec_to);
         else if (which.isDateTime()) executeIntType<UInt32, first>(icolumn, vec_to);
         /// TODO: executeIntType() for Decimal32/64 leads to incompatible result
         else if (which.isDecimal32()) executeBigIntType<Decimal32, first>(icolumn, vec_to);

src/Functions/FunctionsJSON.h

@@ -607,6 +607,8 @@ public:
     }
 };
 
+template <typename JSONParser>
+class JSONExtractRawImpl;
 
 /// Nodes of the extract tree. We need the extract tree to extract from JSON complex values containing array, tuples or nullables.
 template <typename JSONParser>
@@ -691,7 +693,10 @@ struct JSONExtractTree
     public:
         bool insertResultToColumn(IColumn & dest, const Element & element) override
         {
-            return JSONExtractStringImpl<JSONParser>::insertResultToColumn(dest, element, {});
+            if (element.isString())
+                return JSONExtractStringImpl<JSONParser>::insertResultToColumn(dest, element, {});
+            else
+                return JSONExtractRawImpl<JSONParser>::insertResultToColumn(dest, element, {});
         }
     };
 

src/Functions/GatherUtils/Sources.h

@@ -755,6 +755,7 @@ struct GenericValueSource : public ValueSourceImpl<GenericValueSource>
 {
     using Slice = GenericValueSlice;
     using SinkType = GenericArraySink;
+    using Column = IColumn;
 
     const IColumn * column;
     size_t total_rows;

src/Functions/IFunction.h

@@ -113,7 +113,8 @@ public:
 
     virtual ~IFunctionBase() = default;
 
-    virtual ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run = false) const
+    virtual ColumnPtr execute(
+        const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run = false) const
     {
         return prepare(arguments)->execute(arguments, result_type, input_rows_count, dry_run);
     }
@@ -161,7 +162,8 @@ public:
       * Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForConstants,
       * useDefaultImplementationForLowCardinality are not applied.
       */
-    virtual ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & /* arguments */, const DataTypePtr & /* result_type */) const { return nullptr; }
+    virtual ColumnPtr getConstantResultForNonConstArguments(
+        const ColumnsWithTypeAndName & /* arguments */, const DataTypePtr & /* result_type */) const { return nullptr; }
 
     /** Function is called "injective" if it returns different result for different values of arguments.
       * Example: hex, negate, tuple...
@@ -358,6 +360,10 @@ public:
       */
     virtual bool useDefaultImplementationForConstants() const { return false; }
 
+    /** Some arguments could remain constant during this implementation.
+      */
+    virtual ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return {}; }
+
     /** If function arguments has single low cardinality column and all other arguments are constants, call function on nested column.
       * Otherwise, convert all low cardinality columns to ordinary columns.
       * Returns ColumnLowCardinality if at least one argument is ColumnLowCardinality.
@@ -367,10 +373,6 @@ public:
     /// If it isn't, will convert all ColumnLowCardinality arguments to full columns.
     virtual bool canBeExecutedOnLowCardinalityDictionary() const { return true; }
 
-    /** Some arguments could remain constant during this implementation.
-      */
-    virtual ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return {}; }
-
     /** True if function can be called on default arguments (include Nullable's) and won't throw.
       * Counterexample: modulo(0, 0)
       */

src/Functions/PolygonUtils.h

@@ -6,6 +6,7 @@
 #include <Columns/IColumn.h>
 #include <Columns/ColumnVector.h>
 #include <Common/typeid_cast.h>
+#include <Common/NaNUtils.h>
 #include <Common/SipHash.h>
 #include <common/range.h>
 
@@ -40,6 +41,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
+    extern const int BAD_ARGUMENTS;
 }
 
 
@@ -304,6 +306,13 @@ void PointInPolygonWithGrid<CoordinateType>::calcGridAttributes(
     y_scale = 1 / cell_height;
     x_shift = -min_corner.x();
     y_shift = -min_corner.y();
+
+    if (!(isFinite(x_scale)
+        && isFinite(y_scale)
+        && isFinite(x_shift)
+        && isFinite(y_shift)
+        && isFinite(grid_size)))
+        throw Exception("Polygon is not valid: bounding box is unbounded", ErrorCodes::BAD_ARGUMENTS);
 }
 
 template <typename CoordinateType>
@@ -358,7 +367,7 @@ bool PointInPolygonWithGrid<CoordinateType>::contains(CoordinateType x, Coordina
     if (has_empty_bound)
         return false;
 
-    if (std::isnan(x) || std::isnan(y))
+    if (!isFinite(x) || !isFinite(y))
         return false;
 
     CoordinateType float_row = (y + y_shift) * y_scale;

src/Functions/URL/FirstSignificantSubdomainCustomImpl.h

@@ -41,6 +41,9 @@ public:
     String getName() const override { return name; }
     size_t getNumberOfArguments() const override { return 2; }
 
+    bool useDefaultImplementationForConstants() const override { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
+
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         if (!isString(arguments[0].type))
@@ -65,9 +68,7 @@ public:
         const ColumnConst * column_tld_list_name = checkAndGetColumnConstStringOrFixedString(arguments[1].column.get());
         FirstSignificantSubdomainCustomLookup tld_lookup(column_tld_list_name->getValue<String>());
 
-        /// FIXME: convertToFullColumnIfConst() is suboptimal
+        if (const ColumnString * col = checkAndGetColumn<ColumnString>(*arguments[0].column))
-        auto column = arguments[0].column->convertToFullColumnIfConst();
-        if (const ColumnString * col = checkAndGetColumn<ColumnString>(*column))
         {
             auto col_res = ColumnString::create();
             vector(tld_lookup, col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets());

src/Functions/array/arrayElement.cpp

@@ -12,6 +12,7 @@
 #include <Columns/ColumnNullable.h>
 #include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnString.h>
+#include <Columns/ColumnFixedString.h>
 #include <Columns/ColumnTuple.h>
 #include <Columns/ColumnMap.h>
 #include <Common/typeid_cast.h>
@@ -110,6 +111,9 @@ private:
     static bool matchKeyToIndexString(const IColumn & data, const Offsets & offsets,
         const ColumnsWithTypeAndName & arguments, PaddedPODArray<UInt64> & matched_idxs);
 
+    static bool matchKeyToIndexFixedString(const IColumn & data, const Offsets & offsets,
+        const ColumnsWithTypeAndName & arguments, PaddedPODArray<UInt64> & matched_idxs);
+
     static bool matchKeyToIndexStringConst(const IColumn & data, const Offsets & offsets,
         const Field & index, PaddedPODArray<UInt64> & matched_idxs);
 
@@ -767,6 +771,19 @@ struct MatcherString
     }
 };
 
+struct MatcherFixedString
+{
+    const ColumnFixedString & data;
+    const ColumnFixedString & index;
+
+    bool match(size_t row_data, size_t row_index) const
+    {
+        auto data_ref = data.getDataAt(row_data);
+        auto index_ref = index.getDataAt(row_index);
+        return memequalSmallAllowOverflow15(index_ref.data, index_ref.size, data_ref.data, data_ref.size);
+    }
+};
+
 struct MatcherStringConst
 {
     const ColumnString & data;
@@ -863,6 +880,23 @@ bool FunctionArrayElement::matchKeyToIndexString(
     return true;
 }
 
+bool FunctionArrayElement::matchKeyToIndexFixedString(
+    const IColumn & data, const Offsets & offsets,
+    const ColumnsWithTypeAndName & arguments, PaddedPODArray<UInt64> & matched_idxs)
+{
+    const auto * index_string = checkAndGetColumn<ColumnFixedString>(arguments[1].column.get());
+    if (!index_string)
+        return false;
+
+    const auto * data_string = checkAndGetColumn<ColumnFixedString>(&data);
+    if (!data_string)
+        return false;
+
+    MatcherFixedString matcher{*data_string, *index_string};
+    executeMatchKeyToIndex(offsets, matched_idxs, matcher);
+    return true;
+}
+
 template <typename DataType>
 bool FunctionArrayElement::matchKeyToIndexNumberConst(
     const IColumn & data, const Offsets & offsets,
@@ -922,8 +956,10 @@ bool FunctionArrayElement::matchKeyToIndex(
         || matchKeyToIndexNumber<Int64>(data, offsets, arguments, matched_idxs)
         || matchKeyToIndexNumber<Int128>(data, offsets, arguments, matched_idxs)
         || matchKeyToIndexNumber<Int256>(data, offsets, arguments, matched_idxs)
+        || matchKeyToIndexNumber<UInt256>(data, offsets, arguments, matched_idxs)
         || matchKeyToIndexNumber<UUID>(data, offsets, arguments, matched_idxs)
-        || matchKeyToIndexString(data, offsets, arguments, matched_idxs);
+        || matchKeyToIndexString(data, offsets, arguments, matched_idxs)
+        || matchKeyToIndexFixedString(data, offsets, arguments, matched_idxs);
 }
 
 bool FunctionArrayElement::matchKeyToIndexConst(

src/Functions/array/arrayIntersect.cpp

@@ -6,6 +6,7 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypesDecimal.h>
 #include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeDateTime64.h>
 #include <DataTypes/DataTypeNullable.h>
@@ -407,6 +408,9 @@ ColumnPtr FunctionArrayIntersect::executeImpl(const ColumnsWithTypeAndName & arg
     using DateMap = ClearableHashMapWithStackMemory<DataTypeDate::FieldType,
         size_t, DefaultHash<DataTypeDate::FieldType>, INITIAL_SIZE_DEGREE>;
 
+    using Date32Map = ClearableHashMapWithStackMemory<DataTypeDate32::FieldType,
+    size_t, DefaultHash<DataTypeDate32::FieldType>, INITIAL_SIZE_DEGREE>;
+
     using DateTimeMap = ClearableHashMapWithStackMemory<
         DataTypeDateTime::FieldType, size_t,
         DefaultHash<DataTypeDateTime::FieldType>, INITIAL_SIZE_DEGREE>;
@@ -421,6 +425,8 @@ ColumnPtr FunctionArrayIntersect::executeImpl(const ColumnsWithTypeAndName & arg
 
         if (which.isDate())
             result_column = execute<DateMap, ColumnVector<DataTypeDate::FieldType>, true>(arrays, std::move(column));
+        else if (which.isDate32())
+            result_column = execute<Date32Map, ColumnVector<DataTypeDate32::FieldType>, true>(arrays, std::move(column));
         else if (which.isDateTime())
             result_column = execute<DateTimeMap, ColumnVector<DataTypeDateTime::FieldType>, true>(arrays, std::move(column));
         else if (which.isString())

src/Functions/geoToH3.cpp

@@ -21,6 +21,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int INCORRECT_DATA;
 }
 
 namespace
@@ -79,11 +80,14 @@ public:
             const double lat = col_lat->getFloat64(row);
             const UInt8 res = col_res->getUInt(row);
 
-            GeoCoord coord;
+            LatLng coord;
-            coord.lon = degsToRads(lon);
+            coord.lng = degsToRads(lon);
             coord.lat = degsToRads(lat);
 
-            H3Index hindex = geoToH3(&coord, res);
+            H3Index hindex;
+            H3Error err = latLngToCell(&coord, res, &hindex);
+            if (err)
+                throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect coordinates latitude: {}, longitude: {}, error: {}", coord.lat, coord.lng, err);
 
             dst_data[row] = hindex;
         }

src/Functions/h3EdgeAngle.cpp

@@ -66,7 +66,7 @@ public:
                     + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
 
             // Numerical constant is 180 degrees / pi / Earth radius, Earth radius is from h3 sources
-            Float64 res = 8.99320592271288084e-6 * edgeLengthM(resolution);
+            Float64 res = 8.99320592271288084e-6 * getHexagonEdgeLengthAvgM(resolution);
 
             dst_data[row] = res;
         }

src/Functions/h3EdgeLengthM.cpp

@@ -70,7 +70,7 @@ public:
                 throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
                     + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
 
-            Float64 res = edgeLengthM(resolution);
+            Float64 res = getHexagonEdgeLengthAvgM(resolution);
 
             dst_data[row] = res;
         }

src/Functions/h3GetBaseCell.cpp

@@ -59,7 +59,7 @@ public:
         {
             const UInt64 hindex = col_hindex->getUInt(row);
 
-            UInt8 res = h3GetBaseCell(hindex);
+            UInt8 res = getBaseCellNumber(hindex);
 
             dst_data[row] = res;
         }

src/Functions/h3GetResolution.cpp

@@ -59,7 +59,7 @@ public:
         {
             const UInt64 hindex = col_hindex->getUInt(row);
 
-            UInt8 res = h3GetResolution(hindex);
+            UInt8 res = getResolution(hindex);
 
             dst_data[row] = res;
         }

src/Functions/h3HexAreaM2.cpp

@@ -65,7 +65,7 @@ public:
                 throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
                     + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
 
-            Float64 res = hexAreaM2(resolution);
+            Float64 res = getHexagonAreaAvgM2(resolution);
 
             dst_data[row] = res;
         }

src/Functions/h3IndexesAreNeighbors.cpp

@@ -67,7 +67,7 @@ public:
             const UInt64 hindex_origin = col_hindex_origin->getUInt(row);
             const UInt64 hindex_dest = col_hindex_dest->getUInt(row);
 
-            UInt8 res = h3IndexesAreNeighbors(hindex_origin, hindex_dest);
+            UInt8 res = areNeighborCells(hindex_origin, hindex_dest);
 
             dst_data[row] = res;
         }

src/Functions/h3IsValid.cpp

@@ -59,7 +59,7 @@ public:
         {
             const UInt64 hindex = col_hindex->getUInt(row);
 
-            UInt8 is_valid = h3IsValid(hindex) == 0 ? 0 : 1;
+            UInt8 is_valid = isValidCell(hindex) == 0 ? 0 : 1;
 
             dst_data[row] = is_valid;
         }

src/Functions/h3ToChildren.cpp

@@ -84,14 +84,14 @@ public:
                 throw Exception("The argument 'resolution' (" + toString(child_resolution) + ") of function " + getName()
                     + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
 
-            const size_t vec_size = maxH3ToChildrenSize(parent_hindex, child_resolution);
+            const size_t vec_size = cellToChildrenSize(parent_hindex, child_resolution);
             if (vec_size > MAX_ARRAY_SIZE)
                 throw Exception("The result of function" + getName()
                     + " (array of " + toString(vec_size) + " elements) will be too large with resolution argument = "
                     + toString(child_resolution), ErrorCodes::TOO_LARGE_ARRAY_SIZE);
 
             hindex_vec.resize(vec_size);
-            h3ToChildren(parent_hindex, child_resolution, hindex_vec.data());
+            cellToChildren(parent_hindex, child_resolution, hindex_vec.data());
 
             dst_data.reserve(dst_data.size() + vec_size);
             for (auto hindex : hindex_vec)

src/Functions/h3ToParent.cpp

@@ -74,7 +74,7 @@ public:
                 throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
                     + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
 
-            UInt64 res = h3ToParent(hindex, resolution);
+            UInt64 res = cellToParent(hindex, resolution);
 
             dst_data[row] = res;
         }

src/Functions/h3ToString.cpp

@@ -66,7 +66,7 @@ public:
         {
             const UInt64 hindex = col_hindex->getUInt(i);
 
-            if (!h3IsValid(hindex))
+            if (!isValidCell(hindex))
             {
                 throw Exception("Invalid H3 index: " + std::to_string(hindex), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
             }