Merge branch 'master' of github.com:yandex/ClickHouse into async-reads

.github/ISSUE_TEMPLATE/85_bug-report.md

@@ -9,7 +9,7 @@ assignees: ''
 
 > You have to provide the following information whenever possible.
 
-**Describe the bug**
+**Describe what's wrong**
 
 > A clear and concise description of what works not as it is supposed to.
 

CMakeLists.txt

@@ -127,12 +127,13 @@ if (USE_STATIC_LIBRARIES)
     list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
 endif ()
 
-# Implies ${WITH_COVERAGE}
 option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF)
 
 if (ENABLE_FUZZING)
+    # Also set WITH_COVERAGE=1 for better fuzzing process
+    # By default this is disabled, because fuzzers are built in CI with the clickhouse itself.
+    # And we don't want to enable coverage for it.
     message (STATUS "Fuzzing instrumentation enabled")
-    set (WITH_COVERAGE ON)
     set (FUZZER "libfuzzer")
 endif()
 

base/glibc-compatibility/musl/getauxval.c

@@ -1,4 +1,5 @@
 #include <sys/auxv.h>
+#include "atomic.h"
 #include <unistd.h> // __environ
 #include <errno.h>
 
@@ -17,18 +18,7 @@ static size_t __find_auxv(unsigned long type)
     return (size_t) -1;
 }
 
-__attribute__((constructor)) static void __auxv_init()
-{
-    size_t i;
-    for (i = 0; __environ[i]; i++);
-    __auxv = (unsigned long *) (__environ + i + 1);
-
-    size_t secure_idx = __find_auxv(AT_SECURE);
-    if (secure_idx != ((size_t) -1))
-        __auxv_secure = __auxv[secure_idx];
-}
-
-unsigned long getauxval(unsigned long type)
+unsigned long __getauxval(unsigned long type)
 {
     if (type == AT_SECURE)
         return __auxv_secure;
@@ -43,3 +33,38 @@ unsigned long getauxval(unsigned long type)
     errno = ENOENT;
     return 0;
 }
+
+static void * volatile getauxval_func;
+
+static unsigned long  __auxv_init(unsigned long type)
+{
+    if (!__environ)
+    {
+        // __environ is not initialized yet so we can't initialize __auxv right now.
+        // That's normally occurred only when getauxval() is called from some sanitizer's internal code.
+        errno = ENOENT;
+        return 0;
+    }
+
+    // Initialize __auxv and __auxv_secure.
+    size_t i;
+    for (i = 0; __environ[i]; i++);
+    __auxv = (unsigned long *) (__environ + i + 1);
+
+    size_t secure_idx = __find_auxv(AT_SECURE);
+    if (secure_idx != ((size_t) -1))
+        __auxv_secure = __auxv[secure_idx];
+
+    // Now we've initialized __auxv, next time getauxval() will only call __get_auxval().
+    a_cas_p(&getauxval_func, (void *)__auxv_init, (void *)__getauxval);
+
+    return __getauxval(type);
+}
+
+// First time getauxval() will call __auxv_init().
+static void * volatile getauxval_func = (void *)__auxv_init;
+
+unsigned long getauxval(unsigned long type)
+{
+    return ((unsigned long (*)(unsigned long))getauxval_func)(type);
+}

contrib/libunwind

@@ -1 +1 @@
-Subproject commit 6b816d2fba3991f8fd6aaec17d92f68947eab667
+Subproject commit c4ea9848a697747dfa35325af9b3452f30841685

docker/packager/binary/build.sh

@@ -83,6 +83,16 @@ then
     mv "$COMBINED_OUTPUT.tgz" /output
 fi
 
+# Also build fuzzers if any sanitizer specified
+if [ -n "$SANITIZER" ]
+then
+  # Currently we are in build/build_docker directory
+  ../docker/packager/other/fuzzer.sh
+fi
+
+ccache --show-config ||:
+ccache --show-stats ||:
+
 if [ "${CCACHE_DEBUG:-}" == "1" ]
 then
     find . -name '*.ccache-*' -print0 \
@@ -95,4 +105,3 @@ then
     # files in place, and will fail because this directory is not writable.
     tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE"
 fi
-

docker/packager/deb/build.sh

@@ -23,12 +23,24 @@ then
   echo "Place $BINARY_OUTPUT to output"
   mkdir /output/binary ||: # if exists
   mv /build/obj-*/programs/clickhouse* /output/binary
+
   if [ "$BINARY_OUTPUT" = "tests" ]
   then
     mv /build/obj-*/src/unit_tests_dbms /output/binary
   fi
 fi
 
+# Also build fuzzers if any sanitizer specified
+if [ -n "$SANITIZER" ]
+then
+  # Script is supposed that we are in build directory.
+  mkdir -p build/build_docker
+  cd build/build_docker
+  # Launching build script
+  ../docker/packager/other/fuzzer.sh
+  cd
+fi
+
 ccache --show-config ||:
 ccache --show-stats ||:
 

docker/packager/other/fuzzer.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# This script is responsible for building all fuzzers, and copy them to output directory
+# as an archive.
+# Script is supposed that we are in build directory.
+
+set -x -e
+
+printenv
+
+# Delete previous cache, because we add a new flags -DENABLE_FUZZING=1 and -DFUZZER=libfuzzer
+rm -f CMakeCache.txt
+read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
+# Hope, that the most part of files will be in cache, so we just link new executables
+cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" -DENABLE_CLICKHOUSE_ODBC_BRIDGE=OFF \
+    -DENABLE_LIBRARIES=0 -DENABLE_SSL=1 -DUSE_INTERNAL_SSL_LIBRARY=1 -DUSE_UNWIND=ON -DENABLE_EMBEDDED_COMPILER=0 \
+    -DENABLE_EXAMPLES=0 -DENABLE_UTILS=0 -DENABLE_THINLTO=0 "-DSANITIZE=$SANITIZER" \
+    -DENABLE_FUZZING=1 -DFUZZER='libfuzzer' -DENABLE_TCMALLOC=0 -DENABLE_JEMALLOC=0 \
+    -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+
+FUZZER_TARGETS=$(find ../src -name '*_fuzzer.cpp' -execdir basename {} .cpp ';' | tr '\n' ' ')
+
+mkdir -p /output/fuzzers
+for FUZZER_TARGET in $FUZZER_TARGETS
+do
+    # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
+    ninja $NINJA_FLAGS $FUZZER_TARGET
+    # Find this binary in build directory and strip it
+    FUZZER_PATH=$(find ./src -name "$FUZZER_TARGET")
+    strip --strip-unneeded "$FUZZER_PATH"
+    mv "$FUZZER_PATH" /output/fuzzers
+done
+
+tar -zcvf /output/fuzzers.tar.gz /output/fuzzers
+rm -rf /output/fuzzers

docker/packager/packager

@@ -105,6 +105,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
     if image_type == "deb" or image_type == "unbundled":
         result.append("DEB_CC={}".format(cc))
         result.append("DEB_CXX={}".format(cxx))
+        # For building fuzzers
+        result.append("CC={}".format(cc))
+        result.append("CXX={}".format(cxx))
     elif image_type == "binary":
         result.append("CC={}".format(cc))
         result.append("CXX={}".format(cxx))

docker/test/integration/runner/Dockerfile

@@ -79,8 +79,9 @@ RUN python3 -m pip install \
     pytest-timeout \
     pytest-xdist \
     pytest-repeat \
+    pytz \
     redis \
-    tzlocal \
+    tzlocal==2.1 \
     urllib3 \
     requests-kerberos \
     pyhdfs

docker/test/testflows/runner/Dockerfile

@@ -37,7 +37,7 @@ RUN apt-get update \
 ENV TZ=Europe/Moscow
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.1 docker==5.0.0 dicttoxml kazoo tzlocal python-dateutil numpy
+RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.1 docker==5.0.0 dicttoxml kazoo tzlocal==2.1 pytz python-dateutil numpy
 
 ENV DOCKER_CHANNEL stable
 ENV DOCKER_VERSION 20.10.6

docs/en/development/contrib.md

@@ -5,6 +5,86 @@ toc_title: Third-Party Libraries Used
 
 # Third-Party Libraries Used {#third-party-libraries-used}
 
+The list of third-party libraries:
+
+| Library name | License type |
+|:-|:-|
+| abseil-cpp | [Apache](https://github.com/ClickHouse-Extras/abseil-cpp/blob/4f3b686f86c3ebaba7e4e926e62a79cb1c659a54/LICENSE) |
+| AMQP-CPP | [Apache](https://github.com/ClickHouse-Extras/AMQP-CPP/blob/1a6c51f4ac51ac56610fa95081bd2f349911375a/LICENSE) |
+| arrow | [Apache](https://github.com/ClickHouse-Extras/arrow/blob/078e21bad344747b7656ef2d7a4f7410a0a303eb/LICENSE.txt) |
+| avro | [Apache](https://github.com/ClickHouse-Extras/avro/blob/e43c46e87fd32eafdc09471e95344555454c5ef8/LICENSE.txt) |
+| aws | [Apache](https://github.com/ClickHouse-Extras/aws-sdk-cpp/blob/7d48b2c8193679cc4516e5bd68ae4a64b94dae7d/LICENSE.txt) |
+| aws-c-common | [Apache](https://github.com/ClickHouse-Extras/aws-c-common/blob/736a82d1697c108b04a277e66438a7f4e19b6857/LICENSE) |
+| aws-c-event-stream | [Apache](https://github.com/ClickHouse-Extras/aws-c-event-stream/blob/3bc33662f9ccff4f4cbcf9509cc78c26e022fde0/LICENSE) |
+| aws-checksums | [Apache](https://github.com/ClickHouse-Extras/aws-checksums/blob/519d6d9093819b6cf89ffff589a27ef8f83d0f65/LICENSE) |
+| base64 | [BSD 2-clause](https://github.com/ClickHouse-Extras/Turbo-Base64/blob/af9b331f2b4f30b41c70f3a571ff904a8251c1d3/LICENSE) |
+| boost | [Boost](https://github.com/ClickHouse-Extras/boost/blob/9cf09dbfd55a5c6202dedbdf40781a51b02c2675/LICENSE_1_0.txt) |
+| boringssl | [BSD](https://github.com/ClickHouse-Extras/boringssl/blob/a6a2e2ab3e44d97ce98e51c558e989f211de7eb3/LICENSE) |
+| brotli | [MIT](https://github.com/google/brotli/blob/63be8a99401992075c23e99f7c84de1c653e39e2/LICENSE) |
+| capnproto | [MIT](https://github.com/capnproto/capnproto/blob/a00ccd91b3746ef2ab51d40fe3265829949d1ace/LICENSE) |
+| cassandra | [Apache](https://github.com/ClickHouse-Extras/cpp-driver/blob/eb9b68dadbb4417a2c132ad4a1c2fa76e65e6fc1/LICENSE.txt) |
+| cctz | [Apache](https://github.com/ClickHouse-Extras/cctz/blob/c0f1bcb97fd2782f7c3f972fadd5aad5affac4b8/LICENSE.txt) |
+| cityhash102 | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/cityhash102/COPYING) |
+| cppkafka | [BSD 2-clause](https://github.com/mfontanini/cppkafka/blob/5a119f689f8a4d90d10a9635e7ee2bee5c127de1/LICENSE) |
+| croaring | [Apache](https://github.com/RoaringBitmap/CRoaring/blob/2c867e9f9c9e2a3a7032791f94c4c7ae3013f6e0/LICENSE) |
+| curl | [Apache](https://github.com/curl/curl/blob/3b8bbbbd1609c638a3d3d0acb148a33dedb67be3/docs/LICENSE-MIXING.md) |
+| cyrus-sasl | [BSD 2-clause](https://github.com/ClickHouse-Extras/cyrus-sasl/blob/e6466edfd638cc5073debe941c53345b18a09512/COPYING) |
+| double-conversion | [BSD 3-clause](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) |
+| dragonbox | [Apache](https://github.com/ClickHouse-Extras/dragonbox/blob/923705af6fd953aa948fc175f6020b15f7359838/LICENSE-Apache2-LLVM) |
+| fast_float | [Apache](https://github.com/fastfloat/fast_float/blob/7eae925b51fd0f570ccd5c880c12e3e27a23b86f/LICENSE) |
+| fastops | [MIT](https://github.com/ClickHouse-Extras/fastops/blob/88752a5e03cf34639a4a37a4b41d8b463fffd2b5/LICENSE) |
+| flatbuffers | [Apache](https://github.com/ClickHouse-Extras/flatbuffers/blob/eb3f827948241ce0e701516f16cd67324802bce9/LICENSE.txt) |
+| fmtlib | [Unknown](https://github.com/fmtlib/fmt/blob/c108ee1d590089ccf642fc85652b845924067af2/LICENSE.rst) |
+| gcem | [Apache](https://github.com/kthohr/gcem/blob/8d4f1b5d76ea8f6ff12f3f4f34cda45424556b00/LICENSE) |
+| googletest | [BSD 3-clause](https://github.com/google/googletest/blob/e7e591764baba0a0c3c9ad0014430e7a27331d16/LICENSE) |
+| grpc | [Apache](https://github.com/ClickHouse-Extras/grpc/blob/60c986e15cae70aade721d26badabab1f822fdd6/LICENSE) |
+| h3 | [Apache](https://github.com/ClickHouse-Extras/h3/blob/c7f46cfd71fb60e2fefc90e28abe81657deff735/LICENSE) |
+| hyperscan | [Boost](https://github.com/ClickHouse-Extras/hyperscan/blob/e9f08df0213fc637aac0a5bbde9beeaeba2fe9fa/LICENSE) |
+| icu | [Public Domain](https://github.com/unicode-org/icu/blob/faa2f9f9e1fe74c5ed00eba371d2830134cdbea1/icu4c/LICENSE) |
+| icudata | [Public Domain](https://github.com/ClickHouse-Extras/icudata/blob/f020820388e3faafb44cc643574a2d563dfde572/LICENSE) |
+| jemalloc | [BSD 2-clause](https://github.com/ClickHouse-Extras/jemalloc/blob/e6891d9746143bf2cf617493d880ba5a0b9a3efd/COPYING) |
+| krb5 | [MIT](https://github.com/ClickHouse-Extras/krb5/blob/5149dea4e2be0f67707383d2682b897c14631374/src/lib/gssapi/LICENSE) |
+| libc-headers | [LGPL](https://github.com/ClickHouse-Extras/libc-headers/blob/a720b7105a610acbd7427eea475a5b6810c151eb/LICENSE) |
+| libcpuid | [BSD 2-clause](https://github.com/ClickHouse-Extras/libcpuid/blob/8db3b8d2d32d22437f063ce692a1b9bb15e42d18/COPYING) |
+| libcxx | [Apache](https://github.com/ClickHouse-Extras/libcxx/blob/2fa892f69acbaa40f8a18c6484854a6183a34482/LICENSE.TXT) |
+| libcxxabi | [Apache](https://github.com/ClickHouse-Extras/libcxxabi/blob/df8f1e727dbc9e2bedf2282096fa189dc3fe0076/LICENSE.TXT) |
+| libdivide | [zLib](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) |
+| libfarmhash | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libfarmhash/COPYING) |
+| libgsasl | [LGPL](https://github.com/ClickHouse-Extras/libgsasl/blob/383ee28e82f69fa16ed43b48bd9c8ee5b313ab84/LICENSE) |
+| libhdfs3 | [Apache](https://github.com/ClickHouse-Extras/libhdfs3/blob/095b9d48b400abb72d967cb0539af13b1e3d90cf/LICENSE.txt) |
+| libmetrohash | [Apache](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) |
+| libpq | [Unknown](https://github.com/ClickHouse-Extras/libpq/blob/e071ea570f8985aa00e34f5b9d50a3cfe666327e/COPYRIGHT) |
+| libpqxx | [BSD 3-clause](https://github.com/ClickHouse-Extras/libpqxx/blob/357608d11b7a1961c3fb7db2ef9a5dbb2e87da77/COPYING) |
+| librdkafka | [MIT](https://github.com/ClickHouse-Extras/librdkafka/blob/b8554f1682062c85ba519eb54ef2f90e02b812cb/LICENSE.murmur2) |
+| libunwind | [Apache](https://github.com/ClickHouse-Extras/libunwind/blob/6b816d2fba3991f8fd6aaec17d92f68947eab667/LICENSE.TXT) |
+| libuv | [BSD](https://github.com/ClickHouse-Extras/libuv/blob/e2e9b7e9f978ce8a1367b5fe781d97d1ce9f94ab/LICENSE) |
+| llvm | [Apache](https://github.com/ClickHouse-Extras/llvm/blob/e5751459412bce1391fb7a2e9bbc01e131bf72f1/llvm/LICENSE.TXT) |
+| lz4 | [BSD](https://github.com/lz4/lz4/blob/f39b79fb02962a1cd880bbdecb6dffba4f754a11/LICENSE) |
+| mariadb-connector-c | [LGPL](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/5f4034a3a6376416504f17186c55fe401c6d8e5e/COPYING.LIB) |
+| miniselect | [Boost](https://github.com/danlark1/miniselect/blob/be0af6bd0b6eb044d1acc4f754b229972d99903a/LICENSE_1_0.txt) |
+| msgpack-c | [Boost](https://github.com/msgpack/msgpack-c/blob/46684265d50b5d1b062d4c5c428ba08462844b1d/LICENSE_1_0.txt) |
+| murmurhash | [Public Domain](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/murmurhash/LICENSE) |
+| NuRaft | [Apache](https://github.com/ClickHouse-Extras/NuRaft/blob/7ecb16844af6a9c283ad432d85ecc2e7d1544676/LICENSE) |
+| openldap | [Unknown](https://github.com/ClickHouse-Extras/openldap/blob/0208811b6043ca06fda8631a5e473df1ec515ccb/LICENSE) |
+| orc | [Apache](https://github.com/ClickHouse-Extras/orc/blob/0a936f6bbdb9303308973073f8623b5a8d82eae1/LICENSE) |
+| poco | [Boost](https://github.com/ClickHouse-Extras/poco/blob/7351c4691b5d401f59e3959adfc5b4fa263b32da/LICENSE) |
+| protobuf | [BSD 3-clause](https://github.com/ClickHouse-Extras/protobuf/blob/75601841d172c73ae6bf4ce8121f42b875cdbabd/LICENSE) |
+| rapidjson | [MIT](https://github.com/ClickHouse-Extras/rapidjson/blob/c4ef90ccdbc21d5d5a628d08316bfd301e32d6fa/bin/jsonschema/LICENSE) |
+| re2 | [BSD 3-clause](https://github.com/google/re2/blob/13ebb377c6ad763ca61d12dd6f88b1126bd0b911/LICENSE) |
+| replxx | [BSD 3-clause](https://github.com/ClickHouse-Extras/replxx/blob/c81be6c68b146f15f2096b7ef80e3f21fe27004c/LICENSE.md) |
+| rocksdb | [BSD 3-clause](https://github.com/ClickHouse-Extras/rocksdb/blob/b6480c69bf3ab6e298e0d019a07fd4f69029b26a/LICENSE.leveldb) |
+| s2geometry | [Apache](https://github.com/ClickHouse-Extras/s2geometry/blob/20ea540d81f4575a3fc0aea585aac611bcd03ede/LICENSE) |
+| sentry-native | [MIT](https://github.com/ClickHouse-Extras/sentry-native/blob/94644e92f0a3ff14bd35ed902a8622a2d15f7be4/LICENSE) |
+| simdjson | [Apache](https://github.com/simdjson/simdjson/blob/8df32cea3359cb30120795da6020b3b73da01d38/LICENSE) |
+| snappy | [Public Domain](https://github.com/google/snappy/blob/3f194acb57e0487531c96b97af61dcbd025a78a3/COPYING) |
+| sparsehash-c11 | [BSD 3-clause](https://github.com/sparsehash/sparsehash-c11/blob/cf0bffaa456f23bc4174462a789b90f8b6f5f42f/LICENSE) |
+| stats | [Apache](https://github.com/kthohr/stats/blob/b6dd459c10a88c7ea04693c007e9e35820c5d9ad/LICENSE) |
+| thrift | [Apache](https://github.com/apache/thrift/blob/010ccf0a0c7023fea0f6bf4e4078ebdff7e61982/LICENSE) |
+| unixodbc | [LGPL](https://github.com/ClickHouse-Extras/UnixODBC/blob/b0ad30f7f6289c12b76f04bfb9d466374bb32168/COPYING) |
+| xz | [Public Domain](https://github.com/xz-mirror/xz/blob/869b9d1b4edd6df07f819d360d306251f8147353/COPYING) |
+| zlib-ng | [zLib](https://github.com/ClickHouse-Extras/zlib-ng/blob/6a5e93b9007782115f7f7e5235dedc81c4f1facb/LICENSE.md) |
+| zstd | [BSD](https://github.com/facebook/zstd/blob/a488ba114ec17ea1054b9057c26a046fc122b3b6/LICENSE) |
+
 The list of third-party libraries can be obtained by the following query:
 
 ``` sql
@@ -13,84 +93,6 @@ SELECT library_name, license_type, license_path FROM system.licenses ORDER BY li
 
 [Example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIGxpYnJhcnlfbmFtZSwgbGljZW5zZV90eXBlLCBsaWNlbnNlX3BhdGggRlJPTSBzeXN0ZW0ubGljZW5zZXMgT1JERVIgQlkgbGlicmFyeV9uYW1lIENPTExBVEUgJ2VuJw==)
 
-| library_name | license_type | license_path |
-|:-|:-|:-|
-| abseil-cpp | Apache | /contrib/abseil-cpp/LICENSE |
-| AMQP-CPP | Apache | /contrib/AMQP-CPP/LICENSE |
-| arrow | Apache | /contrib/arrow/LICENSE.txt |
-| avro | Apache | /contrib/avro/LICENSE.txt |
-| aws | Apache | /contrib/aws/LICENSE.txt |
-| aws-c-common | Apache | /contrib/aws-c-common/LICENSE |
-| aws-c-event-stream | Apache | /contrib/aws-c-event-stream/LICENSE |
-| aws-checksums | Apache | /contrib/aws-checksums/LICENSE |
-| base64 | BSD 2-clause | /contrib/base64/LICENSE |
-| boost | Boost | /contrib/boost/LICENSE_1_0.txt |
-| boringssl | BSD | /contrib/boringssl/LICENSE |
-| brotli | MIT | /contrib/brotli/LICENSE |
-| capnproto | MIT | /contrib/capnproto/LICENSE |
-| cassandra | Apache | /contrib/cassandra/LICENSE.txt |
-| cctz | Apache | /contrib/cctz/LICENSE.txt |
-| cityhash102 | MIT | /contrib/cityhash102/COPYING |
-| cppkafka | BSD 2-clause | /contrib/cppkafka/LICENSE |
-| croaring | Apache | /contrib/croaring/LICENSE |
-| curl | Apache | /contrib/curl/docs/LICENSE-MIXING.md |
-| cyrus-sasl | BSD 2-clause | /contrib/cyrus-sasl/COPYING |
-| double-conversion | BSD 3-clause | /contrib/double-conversion/LICENSE |
-| dragonbox | Apache | /contrib/dragonbox/LICENSE-Apache2-LLVM |
-| fast_float | Apache | /contrib/fast_float/LICENSE |
-| fastops | MIT | /contrib/fastops/LICENSE |
-| flatbuffers | Apache | /contrib/flatbuffers/LICENSE.txt |
-| fmtlib | Unknown | /contrib/fmtlib/LICENSE.rst |
-| gcem | Apache | /contrib/gcem/LICENSE |
-| googletest | BSD 3-clause | /contrib/googletest/LICENSE |
-| grpc | Apache | /contrib/grpc/LICENSE |
-| h3 | Apache | /contrib/h3/LICENSE |
-| hyperscan | Boost | /contrib/hyperscan/LICENSE |
-| icu | Public Domain | /contrib/icu/icu4c/LICENSE |
-| icudata | Public Domain | /contrib/icudata/LICENSE |
-| jemalloc | BSD 2-clause | /contrib/jemalloc/COPYING |
-| krb5 | MIT | /contrib/krb5/src/lib/gssapi/LICENSE |
-| libc-headers | LGPL | /contrib/libc-headers/LICENSE |
-| libcpuid | BSD 2-clause | /contrib/libcpuid/COPYING |
-| libcxx | Apache | /contrib/libcxx/LICENSE.TXT |
-| libcxxabi | Apache | /contrib/libcxxabi/LICENSE.TXT |
-| libdivide | zLib | /contrib/libdivide/LICENSE.txt |
-| libfarmhash | MIT | /contrib/libfarmhash/COPYING |
-| libgsasl | LGPL | /contrib/libgsasl/LICENSE |
-| libhdfs3 | Apache | /contrib/libhdfs3/LICENSE.txt |
-| libmetrohash | Apache | /contrib/libmetrohash/LICENSE |
-| libpq | Unknown | /contrib/libpq/COPYRIGHT |
-| libpqxx | BSD 3-clause | /contrib/libpqxx/COPYING |
-| librdkafka | MIT | /contrib/librdkafka/LICENSE.murmur2 |
-| libunwind | Apache | /contrib/libunwind/LICENSE.TXT |
-| libuv | BSD | /contrib/libuv/LICENSE |
-| llvm | Apache | /contrib/llvm/llvm/LICENSE.TXT |
-| lz4 | BSD | /contrib/lz4/LICENSE |
-| mariadb-connector-c | LGPL | /contrib/mariadb-connector-c/COPYING.LIB |
-| miniselect | Boost | /contrib/miniselect/LICENSE_1_0.txt |
-| msgpack-c | Boost | /contrib/msgpack-c/LICENSE_1_0.txt |
-| murmurhash | Public Domain | /contrib/murmurhash/LICENSE |
-| NuRaft | Apache | /contrib/NuRaft/LICENSE |
-| openldap | Unknown | /contrib/openldap/LICENSE |
-| orc | Apache | /contrib/orc/LICENSE |
-| poco | Boost | /contrib/poco/LICENSE |
-| protobuf | BSD 3-clause | /contrib/protobuf/LICENSE |
-| rapidjson | MIT | /contrib/rapidjson/bin/jsonschema/LICENSE |
-| re2 | BSD 3-clause | /contrib/re2/LICENSE |
-| replxx | BSD 3-clause | /contrib/replxx/LICENSE.md |
-| rocksdb | BSD 3-clause | /contrib/rocksdb/LICENSE.leveldb |
-| s2geometry | Apache | /contrib/s2geometry/LICENSE |
-| sentry-native | MIT | /contrib/sentry-native/LICENSE |
-| simdjson | Apache | /contrib/simdjson/LICENSE |
-| snappy | Public Domain | /contrib/snappy/COPYING |
-| sparsehash-c11 | BSD 3-clause | /contrib/sparsehash-c11/LICENSE |
-| stats | Apache | /contrib/stats/LICENSE |
-| thrift | Apache | /contrib/thrift/LICENSE |
-| unixodbc | LGPL | /contrib/unixodbc/COPYING |
-| xz | Public Domain | /contrib/xz/COPYING |
-| zlib-ng | zLib | /contrib/zlib-ng/LICENSE.md |
-| zstd | BSD | /contrib/zstd/LICENSE |
-
 ## Guidelines for adding new third-party libraries and maintaining custom changes in them {#adding-third-party-libraries}
 
 1. All external third-party code should reside in the dedicated directories under `contrib` directory of ClickHouse repo. Prefer Git submodules, when available.

docs/en/operations/configuration-files.md

@@ -7,7 +7,7 @@ toc_title: Configuration Files
 
 ClickHouse supports multi-file configuration management. The main server configuration file is `/etc/clickhouse-server/config.xml` or `/etc/clickhouse-server/config.yaml`. Other files must be in the `/etc/clickhouse-server/config.d` directory. Note, that any configuration file can be written either in XML or YAML, but mixing formats in one file is not supported. For example, you can have main configs as `config.xml` and `users.xml` and write additional files in `config.d` and `users.d` directories in `.yaml`.
 
-All the configuration files should be in XML or YAML formats. All XML files should have the same root element, usually `<yandex>`. As for YAML, `yandex:` should not be present, the parser will insert it automatically.
+All XML files should have the same root element, usually `<yandex>`. As for YAML, `yandex:` should not be present, the parser will insert it automatically.
 
 ## Override {#override}
 
@@ -32,7 +32,7 @@ You can also declare attributes as coming from environment variables by using `f
 
 ## Substitution {#substitution}
 
-The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](../operations/server-configuration-parameters/settings.md)).
+The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](../operations/server-configuration-parameters/settings.md#macros)).
 
 If you want to replace an entire element with a substitution use `include` as element name.
 

docs/en/sql-reference/data-types/date.md

@@ -9,9 +9,9 @@ A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). A
 
 The date value is stored without the time zone.
 
-## Examples {#examples}
+**Example**
 
-**1.** Creating a table with a `DateTime`-type column and inserting data into it:
+Creating a table with a `Date`-type column and inserting data into it:
 
 ``` sql
 CREATE TABLE dt
@@ -23,10 +23,7 @@ ENGINE = TinyLog;
 ```
 
 ``` sql
-INSERT INTO dt Values (1546300800, 1), ('2019-01-01', 2);
-```
-
-``` sql
+INSERT INTO dt VALUES (1546300800, 1), ('2019-01-01', 2);
 SELECT * FROM dt;
 ```
 
@@ -37,11 +34,8 @@ SELECT * FROM dt;
 └────────────┴──────────┘
 ```
 
-## See Also {#see-also}
+**See Also**
 
 -   [Functions for working with dates and times](../../sql-reference/functions/date-time-functions.md)
 -   [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime)
 -   [`DateTime` data type](../../sql-reference/data-types/datetime.md)
-
-
-[Original article](https://clickhouse.tech/docs/en/data_types/date/) <!--hide-->

docs/en/sql-reference/data-types/date32.md

@@ -0,0 +1,40 @@
+---
+toc_priority: 48
+toc_title: Date32
+---
+
+# Date32 {#data_type-datetime32}
+
+A date. Supports the date range same with [Datetime64](../../sql-reference/data-types/datetime64.md). Stored in four bytes as the number of days since 1925-01-01. Allows storing values till 2283-11-11. 
+
+**Examples**
+
+Creating a table with a `Date32`-type column and inserting data into it:
+
+``` sql
+CREATE TABLE new
+(
+    `timestamp` Date32,
+    `event_id` UInt8
+)
+ENGINE = TinyLog;
+```
+
+``` sql
+INSERT INTO new VALUES (4102444800, 1), ('2100-01-01', 2);
+SELECT * FROM new;
+```
+
+``` text
+┌──timestamp─┬─event_id─┐
+│ 2100-01-01 │        1 │
+│ 2100-01-01 │        2 │
+└────────────┴──────────┘
+```
+
+**See Also**
+
+-   [toDate32](../../sql-reference/functions/type-conversion-functions.md#todate32)
+-   [toDate32OrZero](../../sql-reference/functions/type-conversion-functions.md#todate32-or-zero)
+-   [toDate32OrNull](../../sql-reference/functions/type-conversion-functions.md#todate32-or-null) 
+

docs/en/sql-reference/data-types/datetime64.md

@@ -17,7 +17,7 @@ DateTime64(precision, [timezone])
 
 Internally, stores data as a number of ‘ticks’ since epoch start (1970-01-01 00:00:00 UTC) as Int64. The tick resolution is determined by the precision parameter. Additionally, the `DateTime64` type can store time zone that is the same for the entire column, that affects how the values of the `DateTime64` type values are displayed in text format and how the values specified as strings are parsed (‘2020-01-01 05:00:01.000’). The time zone is not stored in the rows of the table (or in resultset), but is stored in the column metadata. See details in [DateTime](../../sql-reference/data-types/datetime.md).
 
-Supported range from January 1, 1925 till December 31, 2283.
+Supported range from January 1, 1925 till November 11, 2283.
 
 ## Examples {#examples}
 

docs/en/sql-reference/functions/type-conversion-functions.md

@@ -152,6 +152,104 @@ Alias: `DATE`.
 
 ## toDateTimeOrNull {#todatetimeornull}
 
+## toDate32 {#todate32}
+
+Converts the argument to the [Date32](../../sql-reference/data-types/date32.md) data type. If the value is outside the range returns the border values supported by `Date32`. If the argument has [Date](../../sql-reference/data-types/date.md) type, borders of `Date` are taken into account.
+
+**Syntax** 
+
+``` sql
+toDate32(expr)
+```
+
+**Arguments** 
+
+-   `expr` — The value. [String](../../sql-reference/data-types/string.md), [UInt32](../../sql-reference/data-types/int-uint.md) or [Date](../../sql-reference/data-types/date.md).
+
+**Returned value**
+
+-   A calendar date.
+
+Type: [Date32](../../sql-reference/data-types/date32.md).
+
+**Example**
+
+1. The value is within the range:
+
+``` sql
+SELECT toDate32('1955-01-01') AS value, toTypeName(value);
+```
+
+``` text
+┌──────value─┬─toTypeName(toDate32('1925-01-01'))─┐
+│ 1955-01-01 │ Date32                             │
+└────────────┴────────────────────────────────────┘
+```
+
+2. The value is outside the range:
+
+``` sql
+SELECT toDate32('1924-01-01') AS value, toTypeName(value);
+```
+
+``` text
+┌──────value─┬─toTypeName(toDate32('1925-01-01'))─┐
+│ 1925-01-01 │ Date32                             │
+└────────────┴────────────────────────────────────┘
+```
+
+3. With `Date`-type argument:
+
+``` sql
+SELECT toDate32(toDate('1924-01-01')) AS value, toTypeName(value);
+```
+
+``` text
+┌──────value─┬─toTypeName(toDate32(toDate('1924-01-01')))─┐
+│ 1970-01-01 │ Date32                                     │
+└────────────┴────────────────────────────────────────────┘
+```
+
+## toDate32OrZero {#todate32-or-zero}
+
+The same as [toDate32](#todate32) but returns the min value of [Date32](../../sql-reference/data-types/date32.md) if invalid argument is received.
+
+**Example**
+
+Query:
+
+``` sql
+SELECT toDate32OrZero('1924-01-01'), toDate32OrZero('');
+```
+
+Result:
+
+``` text
+┌─toDate32OrZero('1924-01-01')─┬─toDate32OrZero('')─┐
+│                   1925-01-01 │         1925-01-01 │
+└──────────────────────────────┴────────────────────┘
+```
+
+## toDate32OrNull {#todate32-or-null}
+
+The same as [toDate32](#todate32) but returns `NULL` if invalid argument is received.
+
+**Example**
+
+Query:
+
+``` sql
+SELECT toDate32OrNull('1955-01-01'), toDate32OrNull('');
+```
+
+Result:
+
+``` text
+┌─toDate32OrNull('1955-01-01')─┬─toDate32OrNull('')─┐
+│                   1955-01-01 │               ᴺᵁᴸᴸ │
+└──────────────────────────────┴────────────────────┘
+```
+
 ## toDecimal(32\|64\|128\|256) {#todecimal3264128256}
 
 Converts `value` to the [Decimal](../../sql-reference/data-types/decimal.md) data type with precision of `S`. The `value` can be a number or a string. The `S` (scale) parameter specifies the number of decimal places.

docs/en/sql-reference/table-functions/index.md

@@ -14,7 +14,7 @@ You can use table functions in:
 
         The method for creating a temporary table that is available only in the current query. The table is deleted when the query finishes.
 
--   [CREATE TABLE AS \<table_function()\>](../../sql-reference/statements/create/table.md) query.
+-   [CREATE TABLE AS table_function()](../../sql-reference/statements/create/table.md) query.
 
         It's one of the methods of creating a table.
 

docs/ru/development/contrib.md

@@ -4,40 +4,95 @@ toc_title: "Используемые сторонние библиотеки"
 ---
 
 
-# Используемые сторонние библиотеки {#ispolzuemye-storonnie-biblioteki}
+# Используемые сторонние библиотеки {#third-party-libraries-used}
 
-| Библиотека          | Лицензия                                                                                                                                     |
-|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
-| base64              | [BSD 2-Clause License](https://github.com/aklomp/base64/blob/a27c565d1b6c676beaf297fe503c4518185666f7/LICENSE)                               |
-| boost               | [Boost Software License 1.0](https://github.com/ClickHouse-Extras/boost-extra/blob/6883b40449f378019aec792f9983ce3afc7ff16e/LICENSE_1_0.txt) |
-| brotli              | [MIT](https://github.com/google/brotli/blob/master/LICENSE)                                                                                  |
-| capnproto           | [MIT](https://github.com/capnproto/capnproto/blob/master/LICENSE)                                                                            |
-| cctz                | [Apache License 2.0](https://github.com/google/cctz/blob/4f9776a310f4952454636363def82c2bf6641d5f/LICENSE.txt)                               |
-| double-conversion   | [BSD 3-Clause License](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE)                    |
-| FastMemcpy          | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE)                                                      |
-| googletest          | [BSD 3-Clause License](https://github.com/google/googletest/blob/master/LICENSE)                                                             |
-| h3                  | [Apache License 2.0](https://github.com/uber/h3/blob/master/LICENSE)                                                                         |
-| hyperscan           | [BSD 3-Clause License](https://github.com/intel/hyperscan/blob/master/LICENSE)                                                               |
-| libcxxabi           | [BSD + MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT)                          |
-| libdivide           | [Zlib License](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt)                                           |
-| libgsasl            | [LGPL v2.1](https://github.com/ClickHouse-Extras/libgsasl/blob/3b8948a4042e34fb00b4fb987535dc9e02e39040/LICENSE)                             |
-| libhdfs3            | [Apache License 2.0](https://github.com/ClickHouse-Extras/libhdfs3/blob/bd6505cbb0c130b0db695305b9a38546fa880e5a/LICENSE.txt)                |
-| libmetrohash        | [Apache License 2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libmetrohash/LICENSE)                                      |
-| libpcg-random       | [Apache License 2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libpcg-random/LICENSE-APACHE.txt)                          |
-| libressl            | [OpenSSL License](https://github.com/ClickHouse-Extras/ssl/blob/master/COPYING)                                                              |
-| librdkafka          | [BSD 2-Clause License](https://github.com/edenhill/librdkafka/blob/363dcad5a23dc29381cc626620e68ae418b3af19/LICENSE)                         |
-| libwidechar_width  | [CC0 1.0 Universal](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libwidechar_width/LICENSE)                                     |
-| llvm                | [BSD 3-Clause License](https://github.com/ClickHouse-Extras/llvm/blob/163def217817c90fb982a6daf384744d8472b92b/llvm/LICENSE.TXT)             |
-| lz4                 | [BSD 2-Clause License](https://github.com/lz4/lz4/blob/c10863b98e1503af90616ae99725ecd120265dfb/LICENSE)                                     |
-| mariadb-connector-c | [LGPL v2.1](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/3.1/COPYING.LIB)                                                   |
-| murmurhash          | [Public Domain](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/murmurhash/LICENSE)                                             |
-| pdqsort             | [Zlib License](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/pdqsort/license.txt)                                             |
-| poco                | [Boost Software License - Version 1.0](https://github.com/ClickHouse-Extras/poco/blob/fe5505e56c27b6ecb0dcbc40c49dc2caf4e9637f/LICENSE)      |
-| protobuf            | [BSD 3-Clause License](https://github.com/ClickHouse-Extras/protobuf/blob/12735370922a35f03999afff478e1c6d7aa917a4/LICENSE)                  |
-| re2                 | [BSD 3-Clause License](https://github.com/google/re2/blob/7cf8b88e8f70f97fd4926b56aa87e7f53b2717e0/LICENSE)                                  |
-| UnixODBC            | [LGPL v2.1](https://github.com/ClickHouse-Extras/UnixODBC/tree/b0ad30f7f6289c12b76f04bfb9d466374bb32168)                                     |
-| zlib-ng             | [Zlib License](https://github.com/ClickHouse-Extras/zlib-ng/blob/develop/LICENSE.md)                                                         |
-| zstd                | [BSD 3-Clause License](https://github.com/facebook/zstd/blob/dev/LICENSE)                                                                    |
+Список сторонних библиотек:
+
+| Библиотека | Тип лицензии |
+|:-|:-|
+| abseil-cpp | [Apache](https://github.com/ClickHouse-Extras/abseil-cpp/blob/4f3b686f86c3ebaba7e4e926e62a79cb1c659a54/LICENSE) |
+| AMQP-CPP | [Apache](https://github.com/ClickHouse-Extras/AMQP-CPP/blob/1a6c51f4ac51ac56610fa95081bd2f349911375a/LICENSE) |
+| arrow | [Apache](https://github.com/ClickHouse-Extras/arrow/blob/078e21bad344747b7656ef2d7a4f7410a0a303eb/LICENSE.txt) |
+| avro | [Apache](https://github.com/ClickHouse-Extras/avro/blob/e43c46e87fd32eafdc09471e95344555454c5ef8/LICENSE.txt) |
+| aws | [Apache](https://github.com/ClickHouse-Extras/aws-sdk-cpp/blob/7d48b2c8193679cc4516e5bd68ae4a64b94dae7d/LICENSE.txt) |
+| aws-c-common | [Apache](https://github.com/ClickHouse-Extras/aws-c-common/blob/736a82d1697c108b04a277e66438a7f4e19b6857/LICENSE) |
+| aws-c-event-stream | [Apache](https://github.com/ClickHouse-Extras/aws-c-event-stream/blob/3bc33662f9ccff4f4cbcf9509cc78c26e022fde0/LICENSE) |
+| aws-checksums | [Apache](https://github.com/ClickHouse-Extras/aws-checksums/blob/519d6d9093819b6cf89ffff589a27ef8f83d0f65/LICENSE) |
+| base64 | [BSD 2-clause](https://github.com/ClickHouse-Extras/Turbo-Base64/blob/af9b331f2b4f30b41c70f3a571ff904a8251c1d3/LICENSE) |
+| boost | [Boost](https://github.com/ClickHouse-Extras/boost/blob/9cf09dbfd55a5c6202dedbdf40781a51b02c2675/LICENSE_1_0.txt) |
+| boringssl | [BSD](https://github.com/ClickHouse-Extras/boringssl/blob/a6a2e2ab3e44d97ce98e51c558e989f211de7eb3/LICENSE) |
+| brotli | [MIT](https://github.com/google/brotli/blob/63be8a99401992075c23e99f7c84de1c653e39e2/LICENSE) |
+| capnproto | [MIT](https://github.com/capnproto/capnproto/blob/a00ccd91b3746ef2ab51d40fe3265829949d1ace/LICENSE) |
+| cassandra | [Apache](https://github.com/ClickHouse-Extras/cpp-driver/blob/eb9b68dadbb4417a2c132ad4a1c2fa76e65e6fc1/LICENSE.txt) |
+| cctz | [Apache](https://github.com/ClickHouse-Extras/cctz/blob/c0f1bcb97fd2782f7c3f972fadd5aad5affac4b8/LICENSE.txt) |
+| cityhash102 | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/cityhash102/COPYING) |
+| cppkafka | [BSD 2-clause](https://github.com/mfontanini/cppkafka/blob/5a119f689f8a4d90d10a9635e7ee2bee5c127de1/LICENSE) |
+| croaring | [Apache](https://github.com/RoaringBitmap/CRoaring/blob/2c867e9f9c9e2a3a7032791f94c4c7ae3013f6e0/LICENSE) |
+| curl | [Apache](https://github.com/curl/curl/blob/3b8bbbbd1609c638a3d3d0acb148a33dedb67be3/docs/LICENSE-MIXING.md) |
+| cyrus-sasl | [BSD 2-clause](https://github.com/ClickHouse-Extras/cyrus-sasl/blob/e6466edfd638cc5073debe941c53345b18a09512/COPYING) |
+| double-conversion | [BSD 3-clause](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) |
+| dragonbox | [Apache](https://github.com/ClickHouse-Extras/dragonbox/blob/923705af6fd953aa948fc175f6020b15f7359838/LICENSE-Apache2-LLVM) |
+| fast_float | [Apache](https://github.com/fastfloat/fast_float/blob/7eae925b51fd0f570ccd5c880c12e3e27a23b86f/LICENSE) |
+| fastops | [MIT](https://github.com/ClickHouse-Extras/fastops/blob/88752a5e03cf34639a4a37a4b41d8b463fffd2b5/LICENSE) |
+| flatbuffers | [Apache](https://github.com/ClickHouse-Extras/flatbuffers/blob/eb3f827948241ce0e701516f16cd67324802bce9/LICENSE.txt) |
+| fmtlib | [Unknown](https://github.com/fmtlib/fmt/blob/c108ee1d590089ccf642fc85652b845924067af2/LICENSE.rst) |
+| gcem | [Apache](https://github.com/kthohr/gcem/blob/8d4f1b5d76ea8f6ff12f3f4f34cda45424556b00/LICENSE) |
+| googletest | [BSD 3-clause](https://github.com/google/googletest/blob/e7e591764baba0a0c3c9ad0014430e7a27331d16/LICENSE) |
+| grpc | [Apache](https://github.com/ClickHouse-Extras/grpc/blob/60c986e15cae70aade721d26badabab1f822fdd6/LICENSE) |
+| h3 | [Apache](https://github.com/ClickHouse-Extras/h3/blob/c7f46cfd71fb60e2fefc90e28abe81657deff735/LICENSE) |
+| hyperscan | [Boost](https://github.com/ClickHouse-Extras/hyperscan/blob/e9f08df0213fc637aac0a5bbde9beeaeba2fe9fa/LICENSE) |
+| icu | [Public Domain](https://github.com/unicode-org/icu/blob/faa2f9f9e1fe74c5ed00eba371d2830134cdbea1/icu4c/LICENSE) |
+| icudata | [Public Domain](https://github.com/ClickHouse-Extras/icudata/blob/f020820388e3faafb44cc643574a2d563dfde572/LICENSE) |
+| jemalloc | [BSD 2-clause](https://github.com/ClickHouse-Extras/jemalloc/blob/e6891d9746143bf2cf617493d880ba5a0b9a3efd/COPYING) |
+| krb5 | [MIT](https://github.com/ClickHouse-Extras/krb5/blob/5149dea4e2be0f67707383d2682b897c14631374/src/lib/gssapi/LICENSE) |
+| libc-headers | [LGPL](https://github.com/ClickHouse-Extras/libc-headers/blob/a720b7105a610acbd7427eea475a5b6810c151eb/LICENSE) |
+| libcpuid | [BSD 2-clause](https://github.com/ClickHouse-Extras/libcpuid/blob/8db3b8d2d32d22437f063ce692a1b9bb15e42d18/COPYING) |
+| libcxx | [Apache](https://github.com/ClickHouse-Extras/libcxx/blob/2fa892f69acbaa40f8a18c6484854a6183a34482/LICENSE.TXT) |
+| libcxxabi | [Apache](https://github.com/ClickHouse-Extras/libcxxabi/blob/df8f1e727dbc9e2bedf2282096fa189dc3fe0076/LICENSE.TXT) |
+| libdivide | [zLib](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) |
+| libfarmhash | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libfarmhash/COPYING) |
+| libgsasl | [LGPL](https://github.com/ClickHouse-Extras/libgsasl/blob/383ee28e82f69fa16ed43b48bd9c8ee5b313ab84/LICENSE) |
+| libhdfs3 | [Apache](https://github.com/ClickHouse-Extras/libhdfs3/blob/095b9d48b400abb72d967cb0539af13b1e3d90cf/LICENSE.txt) |
+| libmetrohash | [Apache](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) |
+| libpq | [Unknown](https://github.com/ClickHouse-Extras/libpq/blob/e071ea570f8985aa00e34f5b9d50a3cfe666327e/COPYRIGHT) |
+| libpqxx | [BSD 3-clause](https://github.com/ClickHouse-Extras/libpqxx/blob/357608d11b7a1961c3fb7db2ef9a5dbb2e87da77/COPYING) |
+| librdkafka | [MIT](https://github.com/ClickHouse-Extras/librdkafka/blob/b8554f1682062c85ba519eb54ef2f90e02b812cb/LICENSE.murmur2) |
+| libunwind | [Apache](https://github.com/ClickHouse-Extras/libunwind/blob/6b816d2fba3991f8fd6aaec17d92f68947eab667/LICENSE.TXT) |
+| libuv | [BSD](https://github.com/ClickHouse-Extras/libuv/blob/e2e9b7e9f978ce8a1367b5fe781d97d1ce9f94ab/LICENSE) |
+| llvm | [Apache](https://github.com/ClickHouse-Extras/llvm/blob/e5751459412bce1391fb7a2e9bbc01e131bf72f1/llvm/LICENSE.TXT) |
+| lz4 | [BSD](https://github.com/lz4/lz4/blob/f39b79fb02962a1cd880bbdecb6dffba4f754a11/LICENSE) |
+| mariadb-connector-c | [LGPL](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/5f4034a3a6376416504f17186c55fe401c6d8e5e/COPYING.LIB) |
+| miniselect | [Boost](https://github.com/danlark1/miniselect/blob/be0af6bd0b6eb044d1acc4f754b229972d99903a/LICENSE_1_0.txt) |
+| msgpack-c | [Boost](https://github.com/msgpack/msgpack-c/blob/46684265d50b5d1b062d4c5c428ba08462844b1d/LICENSE_1_0.txt) |
+| murmurhash | [Public Domain](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/murmurhash/LICENSE) |
+| NuRaft | [Apache](https://github.com/ClickHouse-Extras/NuRaft/blob/7ecb16844af6a9c283ad432d85ecc2e7d1544676/LICENSE) |
+| openldap | [Unknown](https://github.com/ClickHouse-Extras/openldap/blob/0208811b6043ca06fda8631a5e473df1ec515ccb/LICENSE) |
+| orc | [Apache](https://github.com/ClickHouse-Extras/orc/blob/0a936f6bbdb9303308973073f8623b5a8d82eae1/LICENSE) |
+| poco | [Boost](https://github.com/ClickHouse-Extras/poco/blob/7351c4691b5d401f59e3959adfc5b4fa263b32da/LICENSE) |
+| protobuf | [BSD 3-clause](https://github.com/ClickHouse-Extras/protobuf/blob/75601841d172c73ae6bf4ce8121f42b875cdbabd/LICENSE) |
+| rapidjson | [MIT](https://github.com/ClickHouse-Extras/rapidjson/blob/c4ef90ccdbc21d5d5a628d08316bfd301e32d6fa/bin/jsonschema/LICENSE) |
+| re2 | [BSD 3-clause](https://github.com/google/re2/blob/13ebb377c6ad763ca61d12dd6f88b1126bd0b911/LICENSE) |
+| replxx | [BSD 3-clause](https://github.com/ClickHouse-Extras/replxx/blob/c81be6c68b146f15f2096b7ef80e3f21fe27004c/LICENSE.md) |
+| rocksdb | [BSD 3-clause](https://github.com/ClickHouse-Extras/rocksdb/blob/b6480c69bf3ab6e298e0d019a07fd4f69029b26a/LICENSE.leveldb) |
+| s2geometry | [Apache](https://github.com/ClickHouse-Extras/s2geometry/blob/20ea540d81f4575a3fc0aea585aac611bcd03ede/LICENSE) |
+| sentry-native | [MIT](https://github.com/ClickHouse-Extras/sentry-native/blob/94644e92f0a3ff14bd35ed902a8622a2d15f7be4/LICENSE) |
+| simdjson | [Apache](https://github.com/simdjson/simdjson/blob/8df32cea3359cb30120795da6020b3b73da01d38/LICENSE) |
+| snappy | [Public Domain](https://github.com/google/snappy/blob/3f194acb57e0487531c96b97af61dcbd025a78a3/COPYING) |
+| sparsehash-c11 | [BSD 3-clause](https://github.com/sparsehash/sparsehash-c11/blob/cf0bffaa456f23bc4174462a789b90f8b6f5f42f/LICENSE) |
+| stats | [Apache](https://github.com/kthohr/stats/blob/b6dd459c10a88c7ea04693c007e9e35820c5d9ad/LICENSE) |
+| thrift | [Apache](https://github.com/apache/thrift/blob/010ccf0a0c7023fea0f6bf4e4078ebdff7e61982/LICENSE) |
+| unixodbc | [LGPL](https://github.com/ClickHouse-Extras/UnixODBC/blob/b0ad30f7f6289c12b76f04bfb9d466374bb32168/COPYING) |
+| xz | [Public Domain](https://github.com/xz-mirror/xz/blob/869b9d1b4edd6df07f819d360d306251f8147353/COPYING) |
+| zlib-ng | [zLib](https://github.com/ClickHouse-Extras/zlib-ng/blob/6a5e93b9007782115f7f7e5235dedc81c4f1facb/LICENSE.md) |
+| zstd | [BSD](https://github.com/facebook/zstd/blob/a488ba114ec17ea1054b9057c26a046fc122b3b6/LICENSE) |
+
+Список всех сторонних библиотек можно получить с помощью запроса:
+
+``` sql
+SELECT library_name, license_type, license_path FROM system.licenses ORDER BY library_name COLLATE 'en';
+```
+
+[Пример](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIGxpYnJhcnlfbmFtZSwgbGljZW5zZV90eXBlLCBsaWNlbnNlX3BhdGggRlJPTSBzeXN0ZW0ubGljZW5zZXMgT1JERVIgQlkgbGlicmFyeV9uYW1lIENPTExBVEUgJ2VuJw==)
 
 ## Рекомендации по добавлению сторонних библиотек и поддержанию в них пользовательских изменений {#adding-third-party-libraries}
 

docs/ru/operations/configuration-files.md

@@ -6,19 +6,51 @@ toc_title: "Конфигурационные файлы"
 
 # Конфигурационные файлы {#configuration_files}
 
-Основной конфигурационный файл сервера - `config.xml` или `config.yaml`. Он расположен в директории `/etc/clickhouse-server/`.
+ClickHouse поддерживает многофайловое управление конфигурацией. Основной конфигурационный файл сервера — `/etc/clickhouse-server/config.xml` или `/etc/clickhouse-server/config.yaml`. Остальные файлы должны находиться в директории `/etc/clickhouse-server/config.d`. Обратите внимание, что конфигурационные файлы могут быть записаны в форматах XML или YAML, но смешение этих форматов в одном файле не поддерживается. Например, можно хранить основные конфигурационные файлы как `config.xml` и `users.xml`, а дополнительные файлы записать в директории `config.d` и `users.d` в формате `.yaml`.
 
-Отдельные настройки могут быть переопределены в файлах `*.xml` и `*.conf`, а также `.yaml` (для файлов в формате YAML) из директории `config.d` рядом с конфигом.
+Все XML файлы должны иметь одинаковый корневой элемент, обычно `<yandex>`. Для YAML элемент `yandex:` должен отсутствовать, так как парсер вставляет его автоматически.
 
-У элементов этих конфигурационных файлов могут быть указаны атрибуты `replace` или `remove`.
+## Переопределение {#override}
 
-Если ни один не указан - объединить содержимое элементов рекурсивно с заменой значений совпадающих детей.
+Некоторые настройки, определенные в основном конфигурационном файле, могут быть переопределены в других файлах:
 
-Если указано `replace` - заменить весь элемент на указанный.
+-   У элементов этих конфигурационных файлов могут быть указаны атрибуты `replace` или `remove`.
+-   Если ни один атрибут не указан, сервер объединит содержимое элементов рекурсивно, заменяя совпадающие значения дочерних элементов.
+-   Если указан атрибут `replace`, сервер заменит весь элемент на указанный.
+-   Если указан атрибут `remove`, сервер удалит элемент.
 
-Если указано `remove` - удалить элемент.
+Также возможно указать атрибуты как переменные среды с помощью `from_env="VARIABLE_NAME"`:
 
-Также в конфиге могут быть указаны «подстановки». Если у элемента присутствует атрибут `incl`, то в качестве значения будет использована соответствующая подстановка из файла. По умолчанию, путь к файлу с подстановками - `/etc/metrika.xml`. Он может быть изменён в конфигурации сервера в элементе [include_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from). Значения подстановок указываются в элементах `/yandex/имя_подстановки` этого файла. Если подстановка, заданная в `incl` отсутствует, то в лог попадает соответствующая запись. Чтобы ClickHouse не писал в лог об отсутствии подстановки, необходимо указать атрибут `optional="true"` (например, настройка [macros](server-configuration-parameters/settings.md)).
+```xml
+<yandex>
+    <macros>
+        <replica from_env="REPLICA" />
+        <layer from_env="LAYER" />
+        <shard from_env="SHARD" />
+    </macros>
+</yandex>
+```
+
+## Подстановки {#substitution}
+
+В конфигурационном файле могут быть указаны «подстановки». Если у элемента присутствует атрибут `incl`, то в качестве значения будет использована соответствующая подстановка из файла. По умолчанию путь к файлу с подстановками - `/etc/metrika.xml`. Он может быть изменён в конфигурации сервера в элементе [include_from](server-configuration-parameters/settings.md#server_configuration_parameters-include_from). Значения подстановок указываются в элементах `/yandex/имя_подстановки` этого файла. Если подстановка, заданная в `incl`, отсутствует, то делается соответствующая запись в лог. Чтобы ClickHouse фиксировал в логе отсутствие подстановки, необходимо указать атрибут `optional="true"` (например, настройки для [macros](server-configuration-parameters/settings.md#macros)).
+
+Если нужно заменить весь элемент подстановкой, можно использовать `include` как имя элемента.
+
+Пример подстановки XML:
+
+```xml
+<yandex>
+    <!-- Appends XML subtree found at `/profiles-in-zookeeper` ZK path to `<profiles>` element. -->
+    <profiles from_zk="/profiles-in-zookeeper" />
+
+    <users>
+        <!-- Replaces `include` element with the subtree found at `/users-in-zookeeper` ZK path. -->
+        <include from_zk="/users-in-zookeeper" />
+        <include from_zk="/other-users-in-zookeeper" />
+    </users>
+</yandex>
+```
 
 Подстановки могут также выполняться из ZooKeeper. Для этого укажите у элемента атрибут `from_zk = "/path/to/node"`. Значение элемента заменится на содержимое узла `/path/to/node` в ZooKeeper. В ZooKeeper-узел также можно положить целое XML-поддерево, оно будет целиком вставлено в исходный элемент.
 
@@ -115,3 +147,9 @@ seq:
 <seq attr1="value1" attr2="value2">123</seq>
 <seq attr1="value1" attr2="value2">abc</seq>
 ```
+
+## Детали реализации {#implementation-details}
+
+При старте сервера для каждого конфигурационного файла создаются файлы предобработки `file-preprocessed.xml`. Они содержат все выполненные подстановки и переопределения (эти сведения записываются просто для информации). Если в конфигурационном файле настроены подстановки ZooKeeper, но при старте сервера ZooKeeper не доступен, то сервер загружает конфигурацию из соответствующего файла предобработки.
+
+Сервер отслеживает как изменения в конфигурационных файлах, так и файлы и узы ZooKeeper, которые были использованы при выполнении подстановок и переопределений, и на ходу перезагружает настройки для пользователей и кластеров. Это означает, что можно изменять кластеры, пользователей и их настройки без перезапуска сервера.

docs/ru/sql-reference/data-types/date.md

@@ -9,9 +9,9 @@ toc_title: Date
 
 Дата хранится без учёта часового пояса.
 
-## Примеры {#examples}
+**Пример**
 
-**1.** Создание таблицы и добавление в неё данных:
+Создание таблицы и добавление в неё данных:
 
 ``` sql
 CREATE TABLE dt
@@ -24,9 +24,6 @@ ENGINE = TinyLog;
 
 ``` sql
 INSERT INTO dt Values (1546300800, 1), ('2019-01-01', 2);
-```
-
-``` sql
 SELECT * FROM dt;
 ```
 
@@ -37,7 +34,7 @@ SELECT * FROM dt;
 └────────────┴──────────┘
 ```
 
-## Смотрите также {#see-also}
+**См. также**
 
 -   [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md)
 -   [Операторы для работы с датой и временем](../../sql-reference/operators/index.md#operators-datetime)

docs/ru/sql-reference/data-types/date32.md

@@ -0,0 +1,40 @@
+---
+toc_priority: 48
+toc_title: Date32
+---
+
+# Date32 {#data_type-datetime32}
+
+Дата. Поддерживается такой же диапазон дат, как для типа [Datetime64](../../sql-reference/data-types/datetime64.md). Значение хранится в четырех байтах и соответствует числу дней с 1925-01-01 по 2283-11-11. 
+
+**Пример**
+
+Создание таблицы со столбцом типа `Date32`и добавление в нее данных:
+
+``` sql
+CREATE TABLE new
+(
+    `timestamp` Date32,
+    `event_id` UInt8
+)
+ENGINE = TinyLog;
+```
+
+``` sql
+INSERT INTO new VALUES (4102444800, 1), ('2100-01-01', 2);
+SELECT * FROM new;
+```
+
+``` text
+┌──timestamp─┬─event_id─┐
+│ 2100-01-01 │        1 │
+│ 2100-01-01 │        2 │
+└────────────┴──────────┘
+```
+
+**См. также**
+
+-   [toDate32](../../sql-reference/functions/type-conversion-functions.md#todate32)
+-   [toDate32OrZero](../../sql-reference/functions/type-conversion-functions.md#todate32-or-zero)
+-   [toDate32OrNull](../../sql-reference/functions/type-conversion-functions.md#todate32-or-null) 
+

docs/ru/sql-reference/data-types/datetime64.md

@@ -17,7 +17,7 @@ DateTime64(precision, [timezone])
 
 Данные хранятся в виде количества ‘тиков’, прошедших с момента начала эпохи (1970-01-01 00:00:00 UTC), в Int64. Размер тика определяется параметром precision. Дополнительно, тип `DateTime64` позволяет хранить часовой пояс, единый для всей колонки, который влияет на то, как будут отображаться значения типа `DateTime64` в текстовом виде и как будут парситься значения заданные в виде строк (‘2020-01-01 05:00:01.000’). Часовой пояс не хранится в строках таблицы (выборки), а хранится в метаданных колонки. Подробнее см. [DateTime](datetime.md).
 
-Поддерживаются значения от 1 января 1925 г. и до 31 декабря 2283 г.
+Поддерживаются значения от 1 января 1925 г. и до 11 ноября 2283 г.
 
 ## Примеры {#examples}
 

docs/ru/sql-reference/functions/type-conversion-functions.md

@@ -152,6 +152,104 @@ Cиноним: `DATE`.
 
 ## toDateTimeOrNull {#todatetimeornull}
 
+## toDate32 {#todate32}
+
+Конвертирует аргумент в значение типа [Date32](../../sql-reference/data-types/date32.md). Если значение выходит за границы диапазона, возвращается пограничное значение `Date32`. Если аргумент имеет тип [Date](../../sql-reference/data-types/date.md), учитываются границы типа `Date`.
+
+**Синтаксис** 
+
+``` sql
+toDate32(value)
+```
+
+**Аргументы** 
+
+-   `value` — Значение даты. [String](../../sql-reference/data-types/string.md), [UInt32](../../sql-reference/data-types/int-uint.md) или [Date](../../sql-reference/data-types/date.md).
+
+**Возвращаемое значение**
+
+-   Календарная дата.
+
+Тип: [Date32](../../sql-reference/data-types/date32.md).
+
+**Пример**
+
+1. Значение находится в границах диапазона:
+
+``` sql
+SELECT toDate32('1955-01-01') AS value, toTypeName(value);
+```
+
+``` text
+┌──────value─┬─toTypeName(toDate32('1925-01-01'))─┐
+│ 1955-01-01 │ Date32                             │
+└────────────┴────────────────────────────────────┘
+```
+
+2. Значение выходит за границы диапазона:
+
+``` sql
+SELECT toDate32('1924-01-01') AS value, toTypeName(value);
+```
+
+``` text
+┌──────value─┬─toTypeName(toDate32('1925-01-01'))─┐
+│ 1925-01-01 │ Date32                             │
+└────────────┴────────────────────────────────────┘
+```
+
+3. С аргументом типа `Date`:
+
+``` sql
+SELECT toDate32(toDate('1924-01-01')) AS value, toTypeName(value);
+```
+
+``` text
+┌──────value─┬─toTypeName(toDate32(toDate('1924-01-01')))─┐
+│ 1970-01-01 │ Date32                                     │
+└────────────┴────────────────────────────────────────────┘
+```
+
+## toDate32OrZero {#todate32-or-zero}
+
+То же самое, что и  [toDate32](#todate32), но возвращает минимальное значение типа [Date32](../../sql-reference/data-types/date32.md), если получен недопустимый аргумент.
+
+**Пример**
+
+Запрос:
+
+``` sql
+SELECT toDate32OrZero('1924-01-01'), toDate32OrZero('');
+```
+
+Результат:
+
+``` text
+┌─toDate32OrZero('1924-01-01')─┬─toDate32OrZero('')─┐
+│                   1925-01-01 │         1925-01-01 │
+└──────────────────────────────┴────────────────────┘
+```
+
+## toDate32OrNull {#todate32-or-null}
+
+То же самое, что и [toDate32](#todate32), но возвращает `NULL`, если получен недопустимый аргумент.
+
+**Пример**
+
+Запрос:
+
+``` sql
+SELECT toDate32OrNull('1955-01-01'), toDate32OrNull('');
+```
+
+Результат:
+
+``` text
+┌─toDate32OrNull('1955-01-01')─┬─toDate32OrNull('')─┐
+│                   1955-01-01 │               ᴺᵁᴸᴸ │
+└──────────────────────────────┴────────────────────┘
+```
+
 ## toDecimal(32\|64\|128\|256) {#todecimal3264128}
 
 Преобразует `value` к типу данных [Decimal](../../sql-reference/functions/type-conversion-functions.md) с точностью `S`. `value` может быть числом или строкой. Параметр `S` (scale) задаёт число десятичных знаков.

docs/zh/getting-started/example-datasets/star-schema.md

@@ -7,7 +7,7 @@ toc_title: Star Schema Benchmark
 
 编译 dbgen:
 
-``` bash
+```bash
 $ git clone git@github.com:vadimtk/ssb-dbgen.git
 $ cd ssb-dbgen
 $ make
@@ -16,9 +16,9 @@ $ make
 开始生成数据：
 
 !!! warning "注意"
-    使用`-s 100`dbgen将生成6亿行数据(67GB), 如果使用`-s 1000`它会生成60亿行数据(这需要很多时间))
+使用`-s 100`dbgen 将生成 6 亿行数据(67GB), 如果使用`-s 1000`它会生成 60 亿行数据(这需要很多时间))
 
-``` bash
+```bash
 $ ./dbgen -s 1000 -T c
 $ ./dbgen -s 1000 -T l
 $ ./dbgen -s 1000 -T p
@@ -26,9 +26,9 @@ $ ./dbgen -s 1000 -T s
 $ ./dbgen -s 1000 -T d
 ```
 
-在ClickHouse中创建数据表：
+在 ClickHouse 中创建数据表：
 
-``` sql
+```sql
 CREATE TABLE customer
 (
         C_CUSTKEY       UInt32,
@@ -93,7 +93,7 @@ ENGINE = MergeTree ORDER BY S_SUPPKEY;
 
 写入数据：
 
-``` bash
+```bash
 $ clickhouse-client --query "INSERT INTO customer FORMAT CSV" < customer.tbl
 $ clickhouse-client --query "INSERT INTO part FORMAT CSV" < part.tbl
 $ clickhouse-client --query "INSERT INTO supplier FORMAT CSV" < supplier.tbl
@@ -102,100 +102,267 @@ $ clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" < lineorder.tbl
 
 将`star schema`转换为`flat schema`：
 
-``` sql
-SET max_memory_usage = 20000000000, allow_experimental_multiple_joins_emulation = 1;
+```sql
+SET max_memory_usage = 20000000000;
 
 CREATE TABLE lineorder_flat
 ENGINE = MergeTree
 PARTITION BY toYear(LO_ORDERDATE)
 ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS
-SELECT l.*, c.*, s.*, p.*
-FROM lineorder l
- ANY INNER JOIN customer c ON (c.C_CUSTKEY = l.LO_CUSTKEY)
- ANY INNER JOIN supplier s ON (s.S_SUPPKEY = l.LO_SUPPKEY)
- ANY INNER JOIN part p ON  (p.P_PARTKEY = l.LO_PARTKEY);
-
-ALTER TABLE lineorder_flat DROP COLUMN C_CUSTKEY, DROP COLUMN S_SUPPKEY, DROP COLUMN P_PARTKEY;
+SELECT
+    l.LO_ORDERKEY AS LO_ORDERKEY,
+    l.LO_LINENUMBER AS LO_LINENUMBER,
+    l.LO_CUSTKEY AS LO_CUSTKEY,
+    l.LO_PARTKEY AS LO_PARTKEY,
+    l.LO_SUPPKEY AS LO_SUPPKEY,
+    l.LO_ORDERDATE AS LO_ORDERDATE,
+    l.LO_ORDERPRIORITY AS LO_ORDERPRIORITY,
+    l.LO_SHIPPRIORITY AS LO_SHIPPRIORITY,
+    l.LO_QUANTITY AS LO_QUANTITY,
+    l.LO_EXTENDEDPRICE AS LO_EXTENDEDPRICE,
+    l.LO_ORDTOTALPRICE AS LO_ORDTOTALPRICE,
+    l.LO_DISCOUNT AS LO_DISCOUNT,
+    l.LO_REVENUE AS LO_REVENUE,
+    l.LO_SUPPLYCOST AS LO_SUPPLYCOST,
+    l.LO_TAX AS LO_TAX,
+    l.LO_COMMITDATE AS LO_COMMITDATE,
+    l.LO_SHIPMODE AS LO_SHIPMODE,
+    c.C_NAME AS C_NAME,
+    c.C_ADDRESS AS C_ADDRESS,
+    c.C_CITY AS C_CITY,
+    c.C_NATION AS C_NATION,
+    c.C_REGION AS C_REGION,
+    c.C_PHONE AS C_PHONE,
+    c.C_MKTSEGMENT AS C_MKTSEGMENT,
+    s.S_NAME AS S_NAME,
+    s.S_ADDRESS AS S_ADDRESS,
+    s.S_CITY AS S_CITY,
+    s.S_NATION AS S_NATION,
+    s.S_REGION AS S_REGION,
+    s.S_PHONE AS S_PHONE,
+    p.P_NAME AS P_NAME,
+    p.P_MFGR AS P_MFGR,
+    p.P_CATEGORY AS P_CATEGORY,
+    p.P_BRAND AS P_BRAND,
+    p.P_COLOR AS P_COLOR,
+    p.P_TYPE AS P_TYPE,
+    p.P_SIZE AS P_SIZE,
+    p.P_CONTAINER AS P_CONTAINER
+FROM lineorder AS l
+INNER JOIN customer AS c ON c.C_CUSTKEY = l.LO_CUSTKEY
+INNER JOIN supplier AS s ON s.S_SUPPKEY = l.LO_SUPPKEY
+INNER JOIN part AS p ON p.P_PARTKEY = l.LO_PARTKEY;
 ```
 
 运行查询:
 
 Q1.1
 
-``` sql
-SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE toYear(LO_ORDERDATE) = 1993 AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25;
+```sql
+SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue
+FROM lineorder_flat
+WHERE toYear(LO_ORDERDATE) = 1993 AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25;
 ```
 
 Q1.2
 
-``` sql
-SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE toYYYYMM(LO_ORDERDATE) = 199401 AND LO_DISCOUNT BETWEEN 4 AND 6 AND LO_QUANTITY BETWEEN 26 AND 35;
+```sql
+SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue
+FROM lineorder_flat
+WHERE toYYYYMM(LO_ORDERDATE) = 199401 AND LO_DISCOUNT BETWEEN 4 AND 6 AND LO_QUANTITY BETWEEN 26 AND 35;
 ```
 
 Q1.3
 
-``` sql
-SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE toISOWeek(LO_ORDERDATE) = 6 AND toYear(LO_ORDERDATE) = 1994 AND LO_DISCOUNT BETWEEN 5 AND 7 AND LO_QUANTITY BETWEEN 26 AND 35;
+```sql
+SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue
+FROM lineorder_flat
+WHERE toISOWeek(LO_ORDERDATE) = 6 AND toYear(LO_ORDERDATE) = 1994
+  AND LO_DISCOUNT BETWEEN 5 AND 7 AND LO_QUANTITY BETWEEN 26 AND 35;
 ```
 
 Q2.1
 
-``` sql
-SELECT sum(LO_REVENUE), toYear(LO_ORDERDATE) AS year, P_BRAND FROM lineorder_flat WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA' GROUP BY year, P_BRAND ORDER BY year, P_BRAND;
+```sql
+SELECT
+    sum(LO_REVENUE),
+    toYear(LO_ORDERDATE) AS year,
+    P_BRAND
+FROM lineorder_flat
+WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA'
+GROUP BY
+    year,
+    P_BRAND
+ORDER BY
+    year,
+    P_BRAND;
 ```
 
 Q2.2
 
-``` sql
-SELECT sum(LO_REVENUE), toYear(LO_ORDERDATE) AS year, P_BRAND FROM lineorder_flat WHERE P_BRAND BETWEEN 'MFGR#2221' AND 'MFGR#2228' AND S_REGION = 'ASIA' GROUP BY year, P_BRAND ORDER BY year, P_BRAND;
+```sql
+SELECT
+    sum(LO_REVENUE),
+    toYear(LO_ORDERDATE) AS year,
+    P_BRAND
+FROM lineorder_flat
+WHERE P_BRAND >= 'MFGR#2221' AND P_BRAND <= 'MFGR#2228' AND S_REGION = 'ASIA'
+GROUP BY
+    year,
+    P_BRAND
+ORDER BY
+    year,
+    P_BRAND;
 ```
 
 Q2.3
 
-``` sql
-SELECT sum(LO_REVENUE), toYear(LO_ORDERDATE) AS year, P_BRAND FROM lineorder_flat WHERE P_BRAND = 'MFGR#2239' AND S_REGION = 'EUROPE' GROUP BY year, P_BRAND ORDER BY year, P_BRAND;
+```sql
+SELECT
+    sum(LO_REVENUE),
+    toYear(LO_ORDERDATE) AS year,
+    P_BRAND
+FROM lineorder_flat
+WHERE P_BRAND = 'MFGR#2239' AND S_REGION = 'EUROPE'
+GROUP BY
+    year,
+    P_BRAND
+ORDER BY
+    year,
+    P_BRAND;
 ```
 
 Q3.1
 
-``` sql
-SELECT C_NATION, S_NATION, toYear(LO_ORDERDATE) AS year, sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE C_REGION = 'ASIA' AND S_REGION = 'ASIA' AND year >= 1992 AND year <= 1997 GROUP BY C_NATION, S_NATION, year ORDER BY year asc, revenue desc;
+```sql
+SELECT
+    C_NATION,
+    S_NATION,
+    toYear(LO_ORDERDATE) AS year,
+    sum(LO_REVENUE) AS revenue
+FROM lineorder_flat
+WHERE C_REGION = 'ASIA' AND S_REGION = 'ASIA' AND year >= 1992 AND year <= 1997
+GROUP BY
+    C_NATION,
+    S_NATION,
+    year
+ORDER BY
+    year ASC,
+    revenue DESC;
 ```
 
 Q3.2
 
-``` sql
-SELECT C_CITY, S_CITY, toYear(LO_ORDERDATE) AS year, sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE C_NATION = 'UNITED STATES' AND S_NATION = 'UNITED STATES' AND year >= 1992 AND year <= 1997 GROUP BY C_CITY, S_CITY, year ORDER BY year asc, revenue desc;
+```sql
+SELECT
+    C_CITY,
+    S_CITY,
+    toYear(LO_ORDERDATE) AS year,
+    sum(LO_REVENUE) AS revenue
+FROM lineorder_flat
+WHERE C_NATION = 'UNITED STATES' AND S_NATION = 'UNITED STATES' AND year >= 1992 AND year <= 1997
+GROUP BY
+    C_CITY,
+    S_CITY,
+    year
+ORDER BY
+    year ASC,
+    revenue DESC;
 ```
 
 Q3.3
 
-``` sql
-SELECT C_CITY, S_CITY, toYear(LO_ORDERDATE) AS year, sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND year >= 1992 AND year <= 1997 GROUP BY C_CITY, S_CITY, year ORDER BY year asc, revenue desc;
+```sql
+SELECT
+    C_CITY,
+    S_CITY,
+    toYear(LO_ORDERDATE) AS year,
+    sum(LO_REVENUE) AS revenue
+FROM lineorder_flat
+WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND year >= 1992 AND year <= 1997
+GROUP BY
+    C_CITY,
+    S_CITY,
+    year
+ORDER BY
+    year ASC,
+    revenue DESC;
 ```
 
 Q3.4
 
-``` sql
-SELECT C_CITY, S_CITY, toYear(LO_ORDERDATE) AS year, sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND toYYYYMM(LO_ORDERDATE) = '199712' GROUP BY C_CITY, S_CITY, year ORDER BY year asc, revenue desc;
+```sql
+SELECT
+    C_CITY,
+    S_CITY,
+    toYear(LO_ORDERDATE) AS year,
+    sum(LO_REVENUE) AS revenue
+FROM lineorder_flat
+WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND toYYYYMM(LO_ORDERDATE) = 199712
+GROUP BY
+    C_CITY,
+    S_CITY,
+    year
+ORDER BY
+    year ASC,
+    revenue DESC;
 ```
 
 Q4.1
 
-``` sql
-SELECT toYear(LO_ORDERDATE) AS year, C_NATION, sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') GROUP BY year, C_NATION ORDER BY year, C_NATION;
+```sql
+SELECT
+    toYear(LO_ORDERDATE) AS year,
+    C_NATION,
+    sum(LO_REVENUE - LO_SUPPLYCOST) AS profit
+FROM lineorder_flat
+WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2')
+GROUP BY
+    year,
+    C_NATION
+ORDER BY
+    year ASC,
+    C_NATION ASC;
 ```
 
 Q4.2
 
-``` sql
-SELECT toYear(LO_ORDERDATE) AS year, S_NATION, P_CATEGORY, sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (year = 1997 OR year = 1998) AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') GROUP BY year, S_NATION, P_CATEGORY ORDER BY year, S_NATION, P_CATEGORY;
+```sql
+SELECT
+    toYear(LO_ORDERDATE) AS year,
+    S_NATION,
+    P_CATEGORY,
+    sum(LO_REVENUE - LO_SUPPLYCOST) AS profit
+FROM lineorder_flat
+WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (year = 1997 OR year = 1998) AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2')
+GROUP BY
+    year,
+    S_NATION,
+    P_CATEGORY
+ORDER BY
+    year ASC,
+    S_NATION ASC,
+    P_CATEGORY ASC;
 ```
 
 Q4.3
 
-``` sql
-SELECT toYear(LO_ORDERDATE) AS year, S_CITY, P_BRAND, sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE S_NATION = 'UNITED STATES' AND (year = 1997 OR year = 1998) AND P_CATEGORY = 'MFGR#14' GROUP BY year, S_CITY, P_BRAND ORDER BY year, S_CITY, P_BRAND;
+```sql
+SELECT
+    toYear(LO_ORDERDATE) AS year,
+    S_CITY,
+    P_BRAND,
+    sum(LO_REVENUE - LO_SUPPLYCOST) AS profit
+FROM lineorder_flat
+WHERE S_NATION = 'UNITED STATES' AND (year = 1997 OR year = 1998) AND P_CATEGORY = 'MFGR#14'
+GROUP BY
+    year,
+    S_CITY,
+    P_BRAND
+ORDER BY
+    year ASC,
+    S_CITY ASC,
+    P_BRAND ASC;
 ```
 
 [原始文章](https://clickhouse.tech/docs/en/getting_started/example_datasets/star_schema/) <!--hide-->

programs/local/LocalServer.cpp

@@ -12,6 +12,7 @@
 #include <Interpreters/executeQuery.h>
 #include <Interpreters/loadMetadata.h>
 #include <Interpreters/DatabaseCatalog.h>
+#include <Interpreters/UserDefinedObjectsLoader.h>
 #include <Interpreters/Session.h>
 #include <Common/Exception.h>
 #include <Common/Macros.h>
@@ -287,6 +288,12 @@ try
         /// Lock path directory before read
         status.emplace(path + "status", StatusFile::write_full_info);
 
+        fs::create_directories(fs::path(path) / "user_defined/");
+        LOG_DEBUG(log, "Loading user defined objects from {}", path);
+        Poco::File(path + "user_defined/").createDirectories();
+        UserDefinedObjectsLoader::instance().loadObjects(global_context);
+        LOG_DEBUG(log, "Loaded user defined objects.");
+
         LOG_DEBUG(log, "Loading metadata from {}", path);
         fs::create_directories(fs::path(path) / "data/");
         fs::create_directories(fs::path(path) / "metadata/");

programs/server/Server.cpp

@@ -53,6 +53,7 @@
 #include <Interpreters/DNSCacheUpdater.h>
 #include <Interpreters/ExternalLoaderXMLConfigRepository.h>
 #include <Interpreters/InterserverCredentials.h>
+#include <Interpreters/UserDefinedObjectsLoader.h>
 #include <Interpreters/JIT/CompiledExpressionCache.h>
 #include <Access/AccessControlManager.h>
 #include <Storages/StorageReplicatedMergeTree.h>
@@ -774,6 +775,7 @@ if (ThreadFuzzer::instance().isEffective())
     {
         fs::create_directories(path / "data/");
         fs::create_directories(path / "metadata/");
+        fs::create_directories(path / "user_defined/");
 
         /// Directory with metadata of tables, which was marked as dropped by Atomic database
         fs::create_directories(path / "metadata_dropped/");
@@ -1083,6 +1085,9 @@ if (ThreadFuzzer::instance().isEffective())
         /// Wait server pool to avoid use-after-free of destroyed context in the handlers
         server_pool.joinAll();
 
+        // Uses a raw pointer to global context for getting ZooKeeper.
+        main_config_reloader.reset();
+
         /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available.
           * At this moment, no one could own shared part of Context.
           */
@@ -1095,6 +1100,18 @@ if (ThreadFuzzer::instance().isEffective())
     /// system logs may copy global context.
     global_context->setCurrentDatabaseNameInGlobalContext(default_database);
 
+    LOG_INFO(log, "Loading user defined objects from {}", path_str);
+    try
+    {
+        UserDefinedObjectsLoader::instance().loadObjects(global_context);
+    }
+    catch (...)
+    {
+        tryLogCurrentException(log, "Caught exception while loading user defined objects");
+        throw;
+    }
+    LOG_DEBUG(log, "Loaded user defined objects");
+
     LOG_INFO(log, "Loading metadata from {}", path_str);
 
     try
@@ -1514,7 +1531,6 @@ if (ThreadFuzzer::instance().isEffective())
                 LOG_INFO(log, "Closed connections.");
 
             dns_cache_updater.reset();
-            main_config_reloader.reset();
 
             if (current_connections)
             {

release

@@ -60,9 +60,6 @@ then
     elif [[ "$SANITIZER" == "thread" ]]; then VERSION_POSTFIX+="+tsan"
     elif [[ "$SANITIZER" == "memory" ]]; then VERSION_POSTFIX+="+msan"
     elif [[ "$SANITIZER" == "undefined" ]]; then VERSION_POSTFIX+="+ubsan"
-    elif [[ "$SANITIZER" == "libfuzzer" ]]; then
-        VERSION_POSTFIX+="+libfuzzer"
-        MALLOC_OPTS="-DENABLE_TCMALLOC=0 -DENABLE_JEMALLOC=0"
     else
         echo "Unknown value of SANITIZER variable: $SANITIZER"
         exit 3

src/Access/AccessType.h

@@ -87,6 +87,7 @@ enum class AccessType
     M(CREATE_DICTIONARY, "", DICTIONARY, CREATE) /* allows to execute {CREATE|ATTACH} DICTIONARY */\
     M(CREATE_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables;
                                                      implicitly enabled by the grant CREATE_TABLE on any table */ \
+    M(CREATE_FUNCTION, "", DATABASE, CREATE) /* allows to execute CREATE FUNCTION */ \
     M(CREATE, "", GROUP, ALL) /* allows to execute {CREATE|ATTACH} */ \
     \
     M(DROP_DATABASE, "", DATABASE, DROP) /* allows to execute {DROP|DETACH} DATABASE */\
@@ -94,6 +95,7 @@ enum class AccessType
     M(DROP_VIEW, "", VIEW, DROP) /* allows to execute {DROP|DETACH} TABLE for views;
                                     implicitly enabled by the grant DROP_TABLE */\
     M(DROP_DICTIONARY, "", DICTIONARY, DROP) /* allows to execute {DROP|DETACH} DICTIONARY */\
+    M(DROP_FUNCTION, "", DATABASE, DROP) /* allows to execute DROP FUNCTION */\
     M(DROP, "", GROUP, ALL) /* allows to execute {DROP|DETACH} */\
     \
     M(TRUNCATE, "TRUNCATE TABLE", TABLE, ALL) \

src/Access/tests/gtest_access_rights_ops.cpp

@@ -45,7 +45,7 @@ TEST(AccessRights, Union)
     lhs.grant(AccessType::INSERT);
     rhs.grant(AccessType::ALL, "db1");
     lhs.makeUnion(rhs);
-    ASSERT_EQ(lhs.toString(), "GRANT INSERT ON *.*, GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
+    ASSERT_EQ(lhs.toString(), "GRANT INSERT ON *.*, GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, CREATE FUNCTION, DROP, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
 }
 
 

src/Common/ErrorCodes.cpp

@@ -580,6 +580,12 @@
     M(609, CANNOT_ADVISE) \
     M(610, UNKNOWN_READ_METHOD) \
     \
+    M(598, FUNCTION_ALREADY_EXISTS) \
+    M(599, CANNOT_DROP_SYSTEM_FUNCTION) \
+    M(600, CANNOT_CREATE_RECURSIVE_FUNCTION) \
+    M(601, OBJECT_ALREADY_STORED_ON_DISK) \
+    M(602, OBJECT_WAS_NOT_STORED_ON_DISK) \
+    \
     M(998, POSTGRESQL_CONNECTION_FAILURE) \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \

src/Compression/CompressedReadBufferBase.cpp

@@ -253,4 +253,3 @@ CompressedReadBufferBase::~CompressedReadBufferBase() = default;    /// Proper d
 
 
 }
-

src/Compression/CompressionCodecDelta.cpp

@@ -82,8 +82,10 @@ void compressDataForType(const char * source, UInt32 source_size, char * dest)
 }
 
 template <typename T>
-void decompressDataForType(const char * source, UInt32 source_size, char * dest)
+void decompressDataForType(const char * source, UInt32 source_size, char * dest, UInt32 output_size)
 {
+    const char * output_end = dest + output_size;
+
     if (source_size % sizeof(T) != 0)
         throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot delta decompress, data size {}  is not aligned to {}", source_size, sizeof(T));
 
@@ -92,6 +94,8 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest)
     while (source < source_end)
     {
         accumulator += unalignedLoad<T>(source);
+        if (dest + sizeof(accumulator) > output_end)
+            throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress the data");
         unalignedStore<T>(dest, accumulator);
 
         source += sizeof(T);
@@ -137,6 +141,7 @@ void CompressionCodecDelta::doDecompressData(const char * source, UInt32 source_
         throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS);
 
     UInt8 bytes_to_skip = uncompressed_size % bytes_size;
+    UInt32 output_size = uncompressed_size - bytes_to_skip;
 
     if (UInt32(2 + bytes_to_skip) > source_size)
         throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS);
@@ -146,16 +151,16 @@ void CompressionCodecDelta::doDecompressData(const char * source, UInt32 source_
     switch (bytes_size)
     {
     case 1:
-        decompressDataForType<UInt8>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]);
+        decompressDataForType<UInt8>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip], output_size);
         break;
     case 2:
-        decompressDataForType<UInt16>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]);
+        decompressDataForType<UInt16>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip], output_size);
         break;
     case 4:
-        decompressDataForType<UInt32>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]);
+        decompressDataForType<UInt32>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip], output_size);
         break;
     case 8:
-        decompressDataForType<UInt64>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]);
+        decompressDataForType<UInt64>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip], output_size);
         break;
     }
 }
@@ -209,4 +214,10 @@ void registerCodecDelta(CompressionCodecFactory & factory)
         return std::make_shared<CompressionCodecDelta>(delta_bytes_size);
     });
 }
+
+CompressionCodecPtr getCompressionCodecDelta(UInt8 delta_bytes_size)
+{
+    return std::make_shared<CompressionCodecDelta>(delta_bytes_size);
+}
+
 }

src/Compression/CompressionCodecDoubleDelta.cpp

@@ -353,12 +353,13 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
 }
 
 template <typename ValueType>
-void decompressDataForType(const char * source, UInt32 source_size, char * dest)
+void decompressDataForType(const char * source, UInt32 source_size, char * dest, UInt32 output_size)
 {
     static_assert(is_unsigned_v<ValueType>, "ValueType must be unsigned.");
     using UnsignedDeltaType = ValueType;
 
     const char * source_end = source + source_size;
+    const char * output_end = dest + output_size;
 
     if (source + sizeof(UInt32) > source_end)
         return;
@@ -374,6 +375,8 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest)
         return;
 
     prev_value = unalignedLoad<ValueType>(source);
+    if (dest + sizeof(prev_value) > output_end)
+        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress the data");
     unalignedStore<ValueType>(dest, prev_value);
 
     source += sizeof(prev_value);
@@ -385,6 +388,8 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest)
 
     prev_delta = unalignedLoad<UnsignedDeltaType>(source);
     prev_value = prev_value + static_cast<ValueType>(prev_delta);
+    if (dest + sizeof(prev_value) > output_end)
+        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress the data");
     unalignedStore<ValueType>(dest, prev_value);
 
     source += sizeof(prev_delta);
@@ -416,6 +421,8 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest)
 
         const UnsignedDeltaType delta = double_delta + prev_delta;
         const ValueType curr_value = prev_value + delta;
+        if (dest + sizeof(curr_value) > output_end)
+            throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress the data");
         unalignedStore<ValueType>(dest, curr_value);
         dest += sizeof(curr_value);
 
@@ -507,6 +514,7 @@ void CompressionCodecDoubleDelta::doDecompressData(const char * source, UInt32 s
         throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS);
 
     UInt8 bytes_to_skip = uncompressed_size % bytes_size;
+    UInt32 output_size = uncompressed_size - bytes_to_skip;
 
     if (UInt32(2 + bytes_to_skip) > source_size)
         throw Exception("Cannot decompress. File has wrong header", ErrorCodes::CANNOT_DECOMPRESS);
@@ -516,16 +524,16 @@ void CompressionCodecDoubleDelta::doDecompressData(const char * source, UInt32 s
     switch (bytes_size)
     {
     case 1:
-        decompressDataForType<UInt8>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]);
+        decompressDataForType<UInt8>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip], output_size);
         break;
     case 2:
-        decompressDataForType<UInt16>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]);
+        decompressDataForType<UInt16>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip], output_size);
         break;
     case 4:
-        decompressDataForType<UInt32>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]);
+        decompressDataForType<UInt32>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip], output_size);
         break;
     case 8:
-        decompressDataForType<UInt64>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip]);
+        decompressDataForType<UInt64>(&source[2 + bytes_to_skip], source_size_no_header, &dest[bytes_to_skip], output_size);
         break;
     }
 }
@@ -543,4 +551,10 @@ void registerCodecDoubleDelta(CompressionCodecFactory & factory)
         return std::make_shared<CompressionCodecDoubleDelta>(data_bytes_size);
     });
 }
+
+CompressionCodecPtr getCompressionCodecDoubleDelta(UInt8 data_bytes_size)
+{
+    return std::make_shared<CompressionCodecDoubleDelta>(data_bytes_size);
+}
+
 }

src/Compression/CompressionCodecEncrypted.h

@@ -51,7 +51,7 @@ namespace DB
           */
         static void setMasterKey(const std::string_view & master_key);
 
-        CompressionCodecEncrypted(const std::string_view & cipher);
+        explicit CompressionCodecEncrypted(const std::string_view & cipher);
 
         uint8_t getMethodByte() const override;
         void updateHash(SipHash & hash) const override;
@@ -88,7 +88,7 @@ namespace DB
           */
         struct KeyHolder : private boost::noncopyable
         {
-            KeyHolder(const std::string_view & master_key);
+            explicit KeyHolder(const std::string_view & master_key);
             ~KeyHolder();
 
             std::string keygen_key;
@@ -99,6 +99,11 @@ namespace DB
 
         static inline std::optional<KeyHolder> keys;
     };
+
+    inline CompressionCodecPtr getCompressionCodecEncrypted(const std::string_view & master_key)
+    {
+        return std::make_shared<CompressionCodecEncrypted>(master_key);
+    }
 }
 
 #endif /* USE_SSL && USE_INTERNAL_SSL_LIBRARY */

src/Compression/CompressionCodecLZ4.cpp

@@ -147,4 +147,10 @@ CompressionCodecLZ4HC::CompressionCodecLZ4HC(int level_)
     setCodecDescription("LZ4HC", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
 }
 
+
+CompressionCodecPtr getCompressionCodecLZ4(int level)
+{
+    return std::make_shared<CompressionCodecLZ4HC>(level);
+}
+
 }

src/Compression/CompressionCodecMultiple.h

@@ -9,7 +9,7 @@ class CompressionCodecMultiple final : public ICompressionCodec
 {
 public:
     CompressionCodecMultiple() = default;   /// Need for CompressionFactory to register codec by method byte.
-    CompressionCodecMultiple(Codecs codecs_);
+    explicit CompressionCodecMultiple(Codecs codecs_);
 
     uint8_t getMethodByte() const override;
 

src/Compression/CompressionCodecZSTD.cpp

@@ -156,4 +156,9 @@ void registerCodecZSTD(CompressionCodecFactory & factory)
     });
 }
 
+CompressionCodecPtr getCompressionCodecZSTD(int level)
+{
+    return std::make_shared<CompressionCodecZSTD>(level);
+}
+
 }

src/Compression/ICompressionCodec.h

@@ -18,6 +18,8 @@ using Codecs = std::vector<CompressionCodecPtr>;
 
 class IDataType;
 
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size);
+
 /**
 * Represents interface for compression codecs like LZ4, ZSTD, etc.
 */
@@ -84,6 +86,8 @@ public:
     virtual bool isNone() const { return false; }
 
 protected:
+    /// This is used for fuzz testing
+    friend int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size);
 
     /// Return size of compressed data without header
     virtual UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const { return uncompressed_size; }

src/Compression/LZ4_decompress_faster.cpp

@@ -450,7 +450,11 @@ bool NO_INLINE decompressImpl(
         const unsigned token = *ip++;
         length = token >> 4;
         if (length == 0x0F)
+        {
+            if (unlikely(ip + 1 >= input_end))
+                return false;
             continue_read_length();
+        }
 
         /// Copy literals.
 
@@ -470,6 +474,20 @@ bool NO_INLINE decompressImpl(
         if (unlikely(copy_end > output_end))
             return false;
 
+        // Due to implementation specifics the copy length is always a multiple of copy_amount
+        size_t real_length = 0;
+
+        static_assert(copy_amount == 8 || copy_amount == 16 || copy_amount == 32);
+        if constexpr (copy_amount == 8)
+            real_length = (((length >> 3) + 1) * 8);
+        else if constexpr (copy_amount == 16)
+            real_length = (((length >> 4) + 1) * 16);
+        else if constexpr (copy_amount == 32)
+            real_length = (((length >> 5) + 1) * 32);
+
+        if (unlikely(ip + real_length >= input_end + ADDITIONAL_BYTES_AT_END_OF_BUFFER))
+             return false;
+
         wildCopy<copy_amount>(op, ip, copy_end);    /// Here we can write up to copy_amount - 1 bytes after buffer.
 
         if (copy_end == output_end)
@@ -494,7 +512,11 @@ bool NO_INLINE decompressImpl(
 
         length = token & 0x0F;
         if (length == 0x0F)
+        {
+            if (unlikely(ip + 1 >= input_end))
+                return false;
             continue_read_length();
+        }
         length += 4;
 
         /// Copy match within block, that produce overlapping pattern. Match may replicate itself.

src/Compression/fuzzers/CMakeLists.txt

@@ -1,2 +1,20 @@
+# Our code has strong cohesion and target associated with `Compression` also depends on `DataTypes`.
+# But we can exclude some files which have dependencies in case of
+# fuzzer related build (we are interested in fuzzing only particular part of our code).
+# So, some symbols will be declared, but not defined. Unfortunately, this trick doesn't work with UBSan.
+# If you want really small size of the resulted binary, just link with fuzz_compression and clickhouse_common_io
+
 add_executable (compressed_buffer_fuzzer compressed_buffer_fuzzer.cpp)
-target_link_libraries (compressed_buffer_fuzzer PRIVATE fuzz_compression clickhouse_common_io ${LIB_FUZZING_ENGINE})
+target_link_libraries (compressed_buffer_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE})
+
+add_executable (lz4_decompress_fuzzer lz4_decompress_fuzzer.cpp)
+target_link_libraries (lz4_decompress_fuzzer PUBLIC dbms lz4 ${LIB_FUZZING_ENGINE})
+
+add_executable (delta_decompress_fuzzer delta_decompress_fuzzer.cpp)
+target_link_libraries (delta_decompress_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE})
+
+add_executable (double_delta_decompress_fuzzer double_delta_decompress_fuzzer.cpp)
+target_link_libraries (double_delta_decompress_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE})
+
+add_executable (encrypted_decompress_fuzzer encrypted_decompress_fuzzer.cpp)
+target_link_libraries (encrypted_decompress_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE})

src/Compression/fuzzers/compressed_buffer_fuzzer.cpp

@@ -17,6 +17,5 @@ try
 }
 catch (...)
 {
-    std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
     return 1;
 }

src/Compression/fuzzers/delta_decompress_fuzzer.cpp

@@ -0,0 +1,44 @@
+#include <iostream>
+#include <string>
+
+#include <Compression/ICompressionCodec.h>
+#include <IO/BufferWithOwnMemory.h>
+
+namespace DB
+{
+    CompressionCodecPtr getCompressionCodecDelta(UInt8 delta_bytes_size);
+}
+
+struct AuxiliaryRandomData
+{
+    UInt8 delta_size_bytes;
+    size_t decompressed_size;
+};
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
+try
+{
+    if (size < sizeof(AuxiliaryRandomData))
+        return 0;
+
+    const auto * p = reinterpret_cast<const AuxiliaryRandomData *>(data);
+    auto codec = DB::getCompressionCodecDelta(p->delta_size_bytes);
+
+    size_t output_buffer_size = p->decompressed_size % 65536;
+    size -= sizeof(AuxiliaryRandomData);
+    data += sizeof(AuxiliaryRandomData) / sizeof(uint8_t);
+
+    // std::string input = std::string(reinterpret_cast<const char*>(data), size);
+    // fmt::print(stderr, "Using input {} of size {}, output size is {}. \n", input, size, output_buffer_size);
+
+    DB::Memory<> memory;
+    memory.resize(output_buffer_size + codec->getAdditionalSizeAtTheEndOfBuffer());
+
+    codec->doDecompressData(reinterpret_cast<const char *>(data), size, memory.data(), output_buffer_size);
+
+    return 0;
+}
+catch (...)
+{
+    return 1;
+}

src/Compression/fuzzers/double_delta_decompress_fuzzer.cpp

@@ -0,0 +1,44 @@
+#include <iostream>
+#include <string>
+
+#include <Compression/ICompressionCodec.h>
+#include <IO/BufferWithOwnMemory.h>
+
+namespace DB
+{
+    CompressionCodecPtr getCompressionCodecDoubleDelta(UInt8 data_bytes_size);
+}
+
+struct AuxiliaryRandomData
+{
+    UInt8 data_bytes_size;
+    size_t decompressed_size;
+};
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
+try
+{
+    if (size < sizeof(AuxiliaryRandomData))
+        return 0;
+
+    const auto * p = reinterpret_cast<const AuxiliaryRandomData *>(data);
+    auto codec = DB::getCompressionCodecDoubleDelta(p->data_bytes_size);
+
+    size_t output_buffer_size = p->decompressed_size % 65536;
+    size -= sizeof(AuxiliaryRandomData);
+    data += sizeof(AuxiliaryRandomData) / sizeof(uint8_t);
+
+    // std::string input = std::string(reinterpret_cast<const char*>(data), size);
+    // fmt::print(stderr, "Using input {} of size {}, output size is {}. \n", input, size, output_buffer_size);
+
+    DB::Memory<> memory;
+    memory.resize(output_buffer_size + codec->getAdditionalSizeAtTheEndOfBuffer());
+
+    codec->doDecompressData(reinterpret_cast<const char *>(data), size, memory.data(), output_buffer_size);
+
+    return 0;
+}
+catch (...)
+{
+    return 1;
+}

src/Compression/fuzzers/encrypted_decompress_fuzzer.cpp

@@ -0,0 +1,52 @@
+#include <iostream>
+#include <string>
+
+#include <Compression/ICompressionCodec.h>
+#include <Compression/CompressionCodecEncrypted.h>
+#include <IO/BufferWithOwnMemory.h>
+
+namespace DB
+{
+    CompressionCodecPtr getCompressionCodecEncrypted(const std::string_view & master_key);
+}
+
+constexpr size_t key_size = 20;
+
+struct AuxiliaryRandomData
+{
+    char key[key_size];
+    size_t decompressed_size;
+};
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
+try
+{
+    if (size < sizeof(AuxiliaryRandomData))
+        return 0;
+
+    const auto * p = reinterpret_cast<const AuxiliaryRandomData *>(data);
+
+    std::string key = std::string(p->key, key_size);
+    auto codec = DB::getCompressionCodecEncrypted(key);
+
+    size_t output_buffer_size = p->decompressed_size % 65536;
+    size -= sizeof(AuxiliaryRandomData);
+    data += sizeof(AuxiliaryRandomData) / sizeof(uint8_t);
+
+    std::string input = std::string(reinterpret_cast<const char*>(data), size);
+    fmt::print(stderr, "Using input {} of size {}, output size is {}. \n", input, size, output_buffer_size);
+
+    if (output_buffer_size < size)
+        return 0;
+
+    DB::Memory<> memory;
+    memory.resize(output_buffer_size + codec->getAdditionalSizeAtTheEndOfBuffer());
+
+    codec->doDecompressData(reinterpret_cast<const char *>(data), size, memory.data(), output_buffer_size);
+
+    return 0;
+}
+catch (...)
+{
+    return 1;
+}

src/Compression/fuzzers/lz4_decompress_fuzzer.cpp

@@ -0,0 +1,47 @@
+#include <iostream>
+#include <string>
+
+#include <Compression/ICompressionCodec.h>
+#include <IO/BufferWithOwnMemory.h>
+#include <Compression/LZ4_decompress_faster.h>
+
+namespace DB
+{
+    CompressionCodecPtr getCompressionCodecLZ4(int level);
+}
+
+struct AuxiliaryRandomData
+{
+    size_t level;
+    size_t decompressed_size;
+};
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
+try
+{
+
+    if (size < sizeof(AuxiliaryRandomData) + LZ4::ADDITIONAL_BYTES_AT_END_OF_BUFFER)
+        return 0;
+
+    const auto * p = reinterpret_cast<const AuxiliaryRandomData *>(data);
+    auto codec = DB::getCompressionCodecLZ4(p->level);
+
+    size_t output_buffer_size = p->decompressed_size % 65536;
+    size -= sizeof(AuxiliaryRandomData);
+    size -= LZ4::ADDITIONAL_BYTES_AT_END_OF_BUFFER;
+    data += sizeof(AuxiliaryRandomData) / sizeof(uint8_t);
+
+    // std::string input = std::string(reinterpret_cast<const char*>(data), size);
+    // fmt::print(stderr, "Using input {} of size {}, output size is {}. \n", input, size, output_buffer_size);
+
+    DB::Memory<> memory;
+    memory.resize(output_buffer_size + LZ4::ADDITIONAL_BYTES_AT_END_OF_BUFFER);
+
+    codec->doDecompressData(reinterpret_cast<const char *>(data), size, memory.data(), output_buffer_size);
+
+    return 0;
+}
+catch (...)
+{
+    return 1;
+}

src/Core/Settings.h

@@ -126,7 +126,7 @@ class IColumn;
     M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \
     M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed queries (shards will process query up to the Complete stage, initiator just proxies the data from the shards). If 2 the initiator will apply ORDER BY and LIMIT stages (it is not in case when shard process query up to the Complete stage)", 0) \
     M(UInt64, distributed_push_down_limit, 1, "If 1, LIMIT will be applied on each shard separatelly. Usually you don't need to use it, since this will be done automatically if it is possible, i.e. for simple query SELECT FROM LIMIT.", 0) \
-    M(Bool, optimize_distributed_group_by_sharding_key, false, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \
+    M(Bool, optimize_distributed_group_by_sharding_key, true, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \
     M(UInt64, optimize_skip_unused_shards_limit, 1000, "Limit for number of sharding key values, turns off optimize_skip_unused_shards if the limit is reached", 0) \
     M(Bool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \
     M(Bool, optimize_skip_unused_shards_rewrite_in, true, "Rewrite IN in query for remote shards to exclude values that does not belong to the shard (requires optimize_skip_unused_shards)", 0) \

src/DataStreams/TTLBlockInputStream.cpp

@@ -76,17 +76,17 @@ TTLBlockInputStream::TTLBlockInputStream(
 
             algorithms.emplace_back(std::make_unique<TTLColumnAlgorithm>(
                 description, old_ttl_infos.columns_ttl[name], current_time_,
-                force_, name, default_expression, default_column_name));
+                force_, name, default_expression, default_column_name, isCompactPart(data_part)));
         }
     }
 
     for (const auto & move_ttl : metadata_snapshot_->getMoveTTLs())
-        algorithms.emplace_back(std::make_unique<TTLMoveAlgorithm>(
-            move_ttl, old_ttl_infos.moves_ttl[move_ttl.result_column], current_time_, force_));
+        algorithms.emplace_back(std::make_unique<TTLUpdateInfoAlgorithm>(
+            move_ttl, TTLUpdateField::MOVES_TTL, move_ttl.result_column, old_ttl_infos.moves_ttl[move_ttl.result_column], current_time_, force_));
 
     for (const auto & recompression_ttl : metadata_snapshot_->getRecompressionTTLs())
-        algorithms.emplace_back(std::make_unique<TTLRecompressionAlgorithm>(
-            recompression_ttl, old_ttl_infos.recompression_ttl[recompression_ttl.result_column], current_time_, force_));
+        algorithms.emplace_back(std::make_unique<TTLUpdateInfoAlgorithm>(
+            recompression_ttl, TTLUpdateField::RECOMPRESSION_TTL, recompression_ttl.result_column, old_ttl_infos.recompression_ttl[recompression_ttl.result_column], current_time_, force_));
 }
 
 Block reorderColumns(Block block, const Block & header)

src/DataStreams/TTLCalcInputStream.cpp

@@ -0,0 +1,77 @@
+#include <DataStreams/TTLCalcInputStream.h>
+#include <DataStreams/TTLUpdateInfoAlgorithm.h>
+
+namespace DB
+{
+
+TTLCalcInputStream::TTLCalcInputStream(
+    const BlockInputStreamPtr & input_,
+    const MergeTreeData & storage_,
+    const StorageMetadataPtr & metadata_snapshot_,
+    const MergeTreeData::MutableDataPartPtr & data_part_,
+    time_t current_time_,
+    bool force_)
+    : data_part(data_part_)
+    , log(&Poco::Logger::get(storage_.getLogName() + " (TTLCalcInputStream)"))
+{
+    children.push_back(input_);
+    header = children.at(0)->getHeader();
+    auto old_ttl_infos = data_part->ttl_infos;
+
+    if (metadata_snapshot_->hasRowsTTL())
+    {
+        const auto & rows_ttl = metadata_snapshot_->getRowsTTL();
+        algorithms.emplace_back(std::make_unique<TTLUpdateInfoAlgorithm>(
+            rows_ttl, TTLUpdateField::TABLE_TTL, rows_ttl.result_column, old_ttl_infos.table_ttl, current_time_, force_));
+    }
+
+    for (const auto & where_ttl : metadata_snapshot_->getRowsWhereTTLs())
+        algorithms.emplace_back(std::make_unique<TTLUpdateInfoAlgorithm>(
+            where_ttl, TTLUpdateField::ROWS_WHERE_TTL, where_ttl.result_column, old_ttl_infos.rows_where_ttl[where_ttl.result_column], current_time_, force_));
+
+    for (const auto & group_by_ttl : metadata_snapshot_->getGroupByTTLs())
+        algorithms.emplace_back(std::make_unique<TTLUpdateInfoAlgorithm>(
+            group_by_ttl, TTLUpdateField::GROUP_BY_TTL, group_by_ttl.result_column, old_ttl_infos.group_by_ttl[group_by_ttl.result_column], current_time_, force_));
+
+    if (metadata_snapshot_->hasAnyColumnTTL())
+    {
+        for (const auto & [name, description] : metadata_snapshot_->getColumnTTLs())
+        {
+            algorithms.emplace_back(std::make_unique<TTLUpdateInfoAlgorithm>(
+                description, TTLUpdateField::COLUMNS_TTL, name, old_ttl_infos.columns_ttl[name], current_time_, force_));
+        }
+    }
+
+    for (const auto & move_ttl : metadata_snapshot_->getMoveTTLs())
+        algorithms.emplace_back(std::make_unique<TTLUpdateInfoAlgorithm>(
+            move_ttl, TTLUpdateField::MOVES_TTL, move_ttl.result_column, old_ttl_infos.moves_ttl[move_ttl.result_column], current_time_, force_));
+
+    for (const auto & recompression_ttl : metadata_snapshot_->getRecompressionTTLs())
+        algorithms.emplace_back(std::make_unique<TTLUpdateInfoAlgorithm>(
+            recompression_ttl, TTLUpdateField::RECOMPRESSION_TTL, recompression_ttl.result_column, old_ttl_infos.recompression_ttl[recompression_ttl.result_column], current_time_, force_));
+}
+
+Block TTLCalcInputStream::readImpl()
+{
+    auto block = children.at(0)->read();
+    for (const auto & algorithm : algorithms)
+        algorithm->execute(block);
+
+    if (!block)
+        return block;
+
+    Block res;
+    for (const auto & col : header)
+        res.insert(block.getByName(col.name));
+
+    return res;
+}
+
+void TTLCalcInputStream::readSuffixImpl()
+{
+    data_part->ttl_infos = {};
+    for (const auto & algorithm : algorithms)
+        algorithm->finalize(data_part);
+}
+
+}

src/DataStreams/TTLCalcInputStream.h

@@ -0,0 +1,44 @@
+#pragma once
+#include <DataStreams/IBlockInputStream.h>
+#include <Storages/MergeTree/MergeTreeData.h>
+#include <Storages/MergeTree/IMergeTreeDataPart.h>
+#include <Core/Block.h>
+#include <Storages/MergeTree/MergeTreeDataPartTTLInfo.h>
+#include <DataStreams/ITTLAlgorithm.h>
+
+#include <common/DateLUT.h>
+
+namespace DB
+{
+
+class TTLCalcInputStream : public IBlockInputStream
+{
+public:
+    TTLCalcInputStream(
+        const BlockInputStreamPtr & input_,
+        const MergeTreeData & storage_,
+        const StorageMetadataPtr & metadata_snapshot_,
+        const MergeTreeData::MutableDataPartPtr & data_part_,
+        time_t current_time,
+        bool force_
+    );
+
+    String getName() const override { return "TTL_CALC"; }
+    Block getHeader() const override { return header; }
+
+protected:
+    Block readImpl() override;
+
+    /// Finalizes ttl infos and updates data part
+    void readSuffixImpl() override;
+
+private:
+    std::vector<TTLAlgorithmPtr> algorithms;
+
+    /// ttl_infos and empty_columns are updating while reading
+    const MergeTreeData::MutableDataPartPtr & data_part;
+    Poco::Logger * log;
+    Block header;
+};
+
+}

src/DataStreams/TTLColumnAlgorithm.cpp

@@ -10,11 +10,13 @@ TTLColumnAlgorithm::TTLColumnAlgorithm(
     bool force_,
     const String & column_name_,
     const ExpressionActionsPtr & default_expression_,
-    const String & default_column_name_)
+    const String & default_column_name_,
+    bool is_compact_part_)
     : ITTLAlgorithm(description_, old_ttl_info_, current_time_, force_)
     , column_name(column_name_)
     , default_expression(default_expression_)
     , default_column_name(default_column_name_)
+    , is_compact_part(is_compact_part_)
 {
     if (!isMinTTLExpired())
     {
@@ -40,7 +42,7 @@ void TTLColumnAlgorithm::execute(Block & block)
         return;
 
     /// Later drop full column
-    if (isMaxTTLExpired())
+    if (isMaxTTLExpired() && !is_compact_part)
         return;
 
     auto default_column = executeExpressionAndGetColumn(default_expression, block, default_column_name);

src/DataStreams/TTLColumnAlgorithm.h

@@ -17,7 +17,9 @@ public:
         bool force_,
         const String & column_name_,
         const ExpressionActionsPtr & default_expression_,
-        const String & default_column_name_);
+        const String & default_column_name_,
+        bool is_compact_part_
+    );
 
     void execute(Block & block) override;
     void finalize(const MutableDataPartPtr & data_part) const override;
@@ -28,6 +30,7 @@ private:
     const String default_column_name;
 
     bool is_fully_empty = true;
+    bool is_compact_part;
 };
 
 }

src/DataStreams/TTLUpdateInfoAlgorithm.cpp

@@ -4,8 +4,15 @@ namespace DB
 {
 
 TTLUpdateInfoAlgorithm::TTLUpdateInfoAlgorithm(
-    const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_)
+    const TTLDescription & description_,
+    const TTLUpdateField ttl_update_field_,
+    const String ttl_update_key_,
+    const TTLInfo & old_ttl_info_,
+    time_t current_time_,
+    bool force_)
     : ITTLAlgorithm(description_, old_ttl_info_, current_time_, force_)
+    , ttl_update_field(ttl_update_field_)
+    , ttl_update_key(ttl_update_key_)
 {
 }
 
@@ -22,26 +29,37 @@ void TTLUpdateInfoAlgorithm::execute(Block & block)
     }
 }
 
-TTLMoveAlgorithm::TTLMoveAlgorithm(
-    const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_)
-    : TTLUpdateInfoAlgorithm(description_, old_ttl_info_, current_time_, force_)
+void TTLUpdateInfoAlgorithm::finalize(const MutableDataPartPtr & data_part) const
 {
-}
+    if (ttl_update_field == TTLUpdateField::RECOMPRESSION_TTL)
+    {
+        data_part->ttl_infos.recompression_ttl[ttl_update_key] = new_ttl_info;
+    }
+    else if (ttl_update_field == TTLUpdateField::MOVES_TTL)
+    {
+        data_part->ttl_infos.moves_ttl[ttl_update_key] = new_ttl_info;
+    }
+    else if (ttl_update_field == TTLUpdateField::GROUP_BY_TTL)
+    {
+        data_part->ttl_infos.group_by_ttl[ttl_update_key] = new_ttl_info;
+        data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max);
+    }
+    else if (ttl_update_field == TTLUpdateField::ROWS_WHERE_TTL)
+    {
+        data_part->ttl_infos.rows_where_ttl[ttl_update_key] = new_ttl_info;
+        data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max);
+    }
+    else if (ttl_update_field == TTLUpdateField::TABLE_TTL)
+    {
+        data_part->ttl_infos.table_ttl = new_ttl_info;
+        data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max);
+    }
+    else if (ttl_update_field == TTLUpdateField::COLUMNS_TTL)
+    {
+        data_part->ttl_infos.columns_ttl[ttl_update_key] = new_ttl_info;
+        data_part->ttl_infos.updatePartMinMaxTTL(new_ttl_info.min, new_ttl_info.max);
+    }
 
-void TTLMoveAlgorithm::finalize(const MutableDataPartPtr & data_part) const
-{
-    data_part->ttl_infos.moves_ttl[description.result_column] = new_ttl_info;
-}
-
-TTLRecompressionAlgorithm::TTLRecompressionAlgorithm(
-    const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_)
-    : TTLUpdateInfoAlgorithm(description_, old_ttl_info_, current_time_, force_)
-{
-}
-
-void TTLRecompressionAlgorithm::finalize(const MutableDataPartPtr & data_part) const
-{
-    data_part->ttl_infos.recompression_ttl[description.result_column] = new_ttl_info;
 }
 
 }

src/DataStreams/TTLUpdateInfoAlgorithm.h

@@ -5,28 +5,35 @@
 namespace DB
 {
 
+enum class TTLUpdateField
+{
+    COLUMNS_TTL,
+    TABLE_TTL,
+    ROWS_WHERE_TTL,
+    MOVES_TTL,
+    RECOMPRESSION_TTL,
+    GROUP_BY_TTL,
+};
+
 /// Calculates new ttl_info and does nothing with data.
 class TTLUpdateInfoAlgorithm : public ITTLAlgorithm
 {
 public:
-    TTLUpdateInfoAlgorithm(const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_);
+    TTLUpdateInfoAlgorithm(
+        const TTLDescription & description_,
+        const TTLUpdateField ttl_update_field_,
+        const String ttl_update_key_,
+        const TTLInfo & old_ttl_info_,
+        time_t current_time_, bool force_
+    );
 
     void execute(Block & block) override;
-    void finalize(const MutableDataPartPtr & data_part) const override = 0;
+    void finalize(const MutableDataPartPtr & data_part) const override;
+
+private:
+    const TTLUpdateField ttl_update_field;
+    const String ttl_update_key;
 };
 
-class TTLMoveAlgorithm final : public TTLUpdateInfoAlgorithm
-{
-public:
-    TTLMoveAlgorithm(const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_);
-    void finalize(const MutableDataPartPtr & data_part) const override;
-};
-
-class TTLRecompressionAlgorithm final : public TTLUpdateInfoAlgorithm
-{
-public:
-    TTLRecompressionAlgorithm(const TTLDescription & description_, const TTLInfo & old_ttl_info_, time_t current_time_, bool force_);
-    void finalize(const MutableDataPartPtr & data_part) const override;
-};
 
 }

src/DataTypes/Serializations/ISerialization.cpp

@@ -1,4 +1,5 @@
 #include <DataTypes/Serializations/ISerialization.h>
+#include <Compression/CompressionFactory.h>
 #include <Columns/IColumn.h>
 #include <IO/WriteHelpers.h>
 #include <IO/Operators.h>

src/Functions/FunctionFactory.cpp

@@ -26,8 +26,8 @@ const String & getFunctionCanonicalNameIfAny(const String & name)
     return FunctionFactory::instance().getCanonicalNameIfAny(name);
 }
 
-void FunctionFactory::registerFunction(const
-    std::string & name,
+void FunctionFactory::registerFunction(
+    const std::string & name,
     Value creator,
     CaseSensitiveness case_sensitiveness)
 {
@@ -119,8 +119,8 @@ FunctionOverloadResolverPtr FunctionFactory::tryGetImpl(
 }
 
 FunctionOverloadResolverPtr FunctionFactory::tryGet(
-        const std::string & name,
-        ContextPtr context) const
+    const std::string & name,
+    ContextPtr context) const
 {
     auto impl = tryGetImpl(name, context);
     return impl ? std::move(impl) : nullptr;

src/Interpreters/InDepthNodeVisitor.h

@@ -16,7 +16,7 @@ class InDepthNodeVisitor
 public:
     using Data = typename Matcher::Data;
 
-    InDepthNodeVisitor(Data & data_, WriteBuffer * ostr_ = nullptr)
+    explicit InDepthNodeVisitor(Data & data_, WriteBuffer * ostr_ = nullptr)
     :   data(data_),
         visit_depth(0),
         ostr(ostr_)

src/Interpreters/InterpreterCreateFunctionQuery.cpp

@@ -0,0 +1,117 @@
+#include <Access/ContextAccess.h>
+#include <Parsers/ASTCreateFunctionQuery.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Interpreters/Context.h>
+#include <Interpreters/ExpressionActions.h>
+#include <Interpreters/ExpressionAnalyzer.h>
+#include <Interpreters/InterpreterCreateFunctionQuery.h>
+#include <Interpreters/FunctionNameNormalizer.h>
+#include <Interpreters/UserDefinedObjectsLoader.h>
+#include <Interpreters/UserDefinedFunctionFactory.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int UNKNOWN_IDENTIFIER;
+    extern const int CANNOT_CREATE_RECURSIVE_FUNCTION;
+    extern const int UNSUPPORTED_METHOD;
+}
+
+BlockIO InterpreterCreateFunctionQuery::execute()
+{
+    auto current_context = getContext();
+    current_context->checkAccess(AccessType::CREATE_FUNCTION);
+
+    FunctionNameNormalizer().visit(query_ptr.get());
+    auto * create_function_query = query_ptr->as<ASTCreateFunctionQuery>();
+
+    if (!create_function_query)
+        throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Expected CREATE FUNCTION query");
+
+    auto & function_name = create_function_query->function_name;
+    validateFunction(create_function_query->function_core, function_name);
+
+    UserDefinedFunctionFactory::instance().registerFunction(function_name, query_ptr);
+
+    if (!is_internal)
+    {
+        try
+        {
+            UserDefinedObjectsLoader::instance().storeObject(current_context, UserDefinedObjectType::Function, function_name, *query_ptr);
+        }
+        catch (Exception & exception)
+        {
+            UserDefinedFunctionFactory::instance().unregisterFunction(function_name);
+            exception.addMessage(fmt::format("while storing user defined function {} on disk", backQuote(function_name)));
+            throw;
+        }
+    }
+
+    return {};
+}
+
+void InterpreterCreateFunctionQuery::validateFunction(ASTPtr function, const String & name)
+{
+    const auto * args_tuple = function->as<ASTFunction>()->arguments->children.at(0)->as<ASTFunction>();
+    std::unordered_set<String> arguments;
+    for (const auto & argument : args_tuple->arguments->children)
+    {
+        const auto & argument_name = argument->as<ASTIdentifier>()->name();
+        auto [_, inserted] = arguments.insert(argument_name);
+        if (!inserted)
+            throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Identifier {} already used as function parameter", argument_name);
+    }
+
+    ASTPtr function_body = function->as<ASTFunction>()->children.at(0)->children.at(1);
+    std::unordered_set<String> identifiers_in_body = getIdentifiers(function_body);
+
+    for (const auto & identifier : identifiers_in_body)
+    {
+        if (!arguments.contains(identifier))
+            throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "Identifier {} does not exist in arguments", backQuote(identifier));
+    }
+
+    validateFunctionRecursiveness(function_body, name);
+}
+
+std::unordered_set<String> InterpreterCreateFunctionQuery::getIdentifiers(ASTPtr node)
+{
+    std::unordered_set<String> identifiers;
+
+    std::stack<ASTPtr> ast_nodes_to_process;
+    ast_nodes_to_process.push(node);
+
+    while (!ast_nodes_to_process.empty())
+    {
+        auto ast_node_to_process = ast_nodes_to_process.top();
+        ast_nodes_to_process.pop();
+
+        for (const auto & child : ast_node_to_process->children)
+        {
+            auto identifier_name_opt = tryGetIdentifierName(child);
+            if (identifier_name_opt)
+                identifiers.insert(identifier_name_opt.value());
+
+            ast_nodes_to_process.push(child);
+        }
+    }
+
+    return identifiers;
+}
+
+void InterpreterCreateFunctionQuery::validateFunctionRecursiveness(ASTPtr node, const String & function_to_create)
+{
+    for (const auto & child : node->children)
+    {
+        auto function_name_opt = tryGetFunctionName(child);
+        if (function_name_opt && function_name_opt.value() == function_to_create)
+            throw Exception(ErrorCodes::CANNOT_CREATE_RECURSIVE_FUNCTION, "You cannot create recursive function");
+
+        validateFunctionRecursiveness(child, function_to_create);
+    }
+}
+
+}

src/Interpreters/InterpreterCreateFunctionQuery.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <Interpreters/IInterpreter.h>
+
+
+namespace DB
+{
+
+class Context;
+
+class InterpreterCreateFunctionQuery : public IInterpreter, WithContext
+{
+public:
+    InterpreterCreateFunctionQuery(const ASTPtr & query_ptr_, ContextPtr context_, bool is_internal_)
+        : WithContext(context_)
+        , query_ptr(query_ptr_)
+        , is_internal(is_internal_) {}
+
+    BlockIO execute() override;
+
+    void setInternal(bool internal_);
+
+private:
+    static void validateFunction(ASTPtr function, const String & name);
+    static std::unordered_set<String> getIdentifiers(ASTPtr node);
+    static void validateFunctionRecursiveness(ASTPtr node, const String & function_to_create);
+
+    ASTPtr query_ptr;
+    bool is_internal;
+};
+
+}

src/Interpreters/InterpreterDropFunctionQuery.cpp

@@ -0,0 +1,27 @@
+#include <Access/ContextAccess.h>
+#include <Interpreters/Context.h>
+#include <Interpreters/FunctionNameNormalizer.h>
+#include <Interpreters/InterpreterDropFunctionQuery.h>
+#include <Interpreters/UserDefinedObjectsLoader.h>
+#include <Interpreters/UserDefinedFunctionFactory.h>
+#include <Parsers/ASTDropFunctionQuery.h>
+
+
+namespace DB
+{
+
+BlockIO InterpreterDropFunctionQuery::execute()
+{
+    auto current_context = getContext();
+    current_context->checkAccess(AccessType::DROP_FUNCTION);
+
+    FunctionNameNormalizer().visit(query_ptr.get());
+    auto & drop_function_query = query_ptr->as<ASTDropFunctionQuery &>();
+
+    UserDefinedFunctionFactory::instance().unregisterFunction(drop_function_query.function_name);
+    UserDefinedObjectsLoader::instance().removeObject(current_context, UserDefinedObjectType::Function, drop_function_query.function_name);
+
+    return {};
+}
+
+}

src/Interpreters/InterpreterDropFunctionQuery.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <Interpreters/IInterpreter.h>
+
+namespace DB
+{
+
+class Context;
+
+class InterpreterDropFunctionQuery : public IInterpreter, WithMutableContext
+{
+public:
+    InterpreterDropFunctionQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) {}
+
+    BlockIO execute() override;
+
+private:
+    ASTPtr query_ptr;
+};
+
+}

src/Interpreters/InterpreterFactory.cpp

@@ -7,7 +7,9 @@
 #include <Parsers/ASTCreateRowPolicyQuery.h>
 #include <Parsers/ASTCreateSettingsProfileQuery.h>
 #include <Parsers/ASTCreateUserQuery.h>
+#include <Parsers/ASTCreateFunctionQuery.h>
 #include <Parsers/ASTDropAccessEntityQuery.h>
+#include <Parsers/ASTDropFunctionQuery.h>
 #include <Parsers/ASTDropQuery.h>
 #include <Parsers/ASTExplainQuery.h>
 #include <Parsers/ASTGrantQuery.h>
@@ -36,6 +38,7 @@
 #include <Interpreters/InterpreterAlterQuery.h>
 #include <Interpreters/InterpreterBackupQuery.h>
 #include <Interpreters/InterpreterCheckQuery.h>
+#include <Interpreters/InterpreterCreateFunctionQuery.h>
 #include <Interpreters/InterpreterCreateQuery.h>
 #include <Interpreters/InterpreterCreateQuotaQuery.h>
 #include <Interpreters/InterpreterCreateRoleQuery.h>
@@ -44,6 +47,7 @@
 #include <Interpreters/InterpreterCreateUserQuery.h>
 #include <Interpreters/InterpreterDescribeQuery.h>
 #include <Interpreters/InterpreterDropAccessEntityQuery.h>
+#include <Interpreters/InterpreterDropFunctionQuery.h>
 #include <Interpreters/InterpreterDropQuery.h>
 #include <Interpreters/InterpreterExistsQuery.h>
 #include <Interpreters/InterpreterExplainQuery.h>
@@ -272,6 +276,14 @@ std::unique_ptr<IInterpreter> InterpreterFactory::get(ASTPtr & query, ContextMut
     {
         return std::make_unique<InterpreterExternalDDLQuery>(query, context);
     }
+    else if (query->as<ASTCreateFunctionQuery>())
+    {
+        return std::make_unique<InterpreterCreateFunctionQuery>(query, context, false /*is_internal*/);
+    }
+    else if (query->as<ASTDropFunctionQuery>())
+    {
+        return std::make_unique<InterpreterDropFunctionQuery>(query, context);
+    }
     else if (query->as<ASTBackupQuery>())
     {
         return std::make_unique<InterpreterBackupQuery>(query, context);

src/Interpreters/MutationsInterpreter.cpp

@@ -156,7 +156,7 @@ ColumnDependencies getAllColumnDependencies(const StorageMetadataPtr & metadata_
     ColumnDependencies dependencies;
     while (!new_updated_columns.empty())
     {
-        auto new_dependencies = metadata_snapshot->getColumnDependencies(new_updated_columns);
+        auto new_dependencies = metadata_snapshot->getColumnDependencies(new_updated_columns, true);
         new_updated_columns.clear();
         for (const auto & dependency : new_dependencies)
         {
@@ -303,6 +303,15 @@ static NameSet getKeyColumns(const StoragePtr & storage, const StorageMetadataPt
     return key_columns;
 }
 
+static bool materializeTTLRecalculateOnly(const StoragePtr & storage)
+{
+    auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast<StorageFromMergeTreeDataPart>(storage);
+    if (!storage_from_merge_tree_data_part)
+        return false;
+
+    return storage_from_merge_tree_data_part->materializeTTLRecalculateOnly();
+}
+
 static void validateUpdateColumns(
     const StoragePtr & storage,
     const StorageMetadataPtr & metadata_snapshot, const NameSet & updated_columns,
@@ -394,8 +403,13 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
     NamesAndTypesList all_columns = columns_desc.getAllPhysical();
 
     NameSet updated_columns;
+    bool materialize_ttl_recalculate_only = materializeTTLRecalculateOnly(storage);
     for (const MutationCommand & command : commands)
     {
+        if (command.type == MutationCommand::Type::UPDATE
+            || command.type == MutationCommand::Type::DELETE)
+            materialize_ttl_recalculate_only = false;
+
         for (const auto & kv : command.column_to_update_expression)
         {
             updated_columns.insert(kv.first);
@@ -569,7 +583,18 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
         else if (command.type == MutationCommand::MATERIALIZE_TTL)
         {
             mutation_kind.set(MutationKind::MUTATE_OTHER);
-            if (metadata_snapshot->hasRowsTTL())
+            if (materialize_ttl_recalculate_only)
+            {
+                // just recalculate ttl_infos without remove expired data
+                auto all_columns_vec = all_columns.getNames();
+                auto new_dependencies = metadata_snapshot->getColumnDependencies(NameSet(all_columns_vec.begin(), all_columns_vec.end()), false);
+                for (const auto & dependency : new_dependencies)
+                {
+                    if (dependency.kind == ColumnDependency::TTL_EXPRESSION)
+                        dependencies.insert(dependency);
+                }
+            }
+            else if (metadata_snapshot->hasRowsTTL())
             {
                 for (const auto & column : all_columns)
                     dependencies.emplace(column.name, ColumnDependency::TTL_TARGET);
@@ -594,19 +619,19 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
                 }
 
                 /// Recalc only skip indices and projections of columns which could be updated by TTL.
-                auto new_dependencies = metadata_snapshot->getColumnDependencies(new_updated_columns);
+                auto new_dependencies = metadata_snapshot->getColumnDependencies(new_updated_columns, true);
                 for (const auto & dependency : new_dependencies)
                 {
                     if (dependency.kind == ColumnDependency::SKIP_INDEX || dependency.kind == ColumnDependency::PROJECTION)
                         dependencies.insert(dependency);
                 }
+            }
 
-                if (dependencies.empty())
-                {
-                    /// Very rare case. It can happen if we have only one MOVE TTL with constant expression.
-                    /// But we still have to read at least one column.
-                    dependencies.emplace(all_columns.front().name, ColumnDependency::TTL_EXPRESSION);
-                }
+            if (dependencies.empty())
+            {
+                /// Very rare case. It can happen if we have only one MOVE TTL with constant expression.
+                /// But we still have to read at least one column.
+                dependencies.emplace(all_columns.front().name, ColumnDependency::TTL_EXPRESSION);
             }
         }
         else if (command.type == MutationCommand::READ_COLUMN)

src/Interpreters/TreeRewriter.cpp

@@ -14,6 +14,7 @@
 #include <Interpreters/CollectJoinOnKeysVisitor.h>
 #include <Interpreters/RequiredSourceColumnsVisitor.h>
 #include <Interpreters/GetAggregatesVisitor.h>
+#include <Interpreters/UserDefinedFunctionsVisitor.h>
 #include <Interpreters/TableJoin.h>
 #include <Interpreters/ExpressionActions.h> /// getSmallestColumn()
 #include <Interpreters/getTableExpressions.h>
@@ -1045,6 +1046,9 @@ TreeRewriterResultPtr TreeRewriter::analyze(
 void TreeRewriter::normalize(
     ASTPtr & query, Aliases & aliases, const NameSet & source_columns_set, bool ignore_alias, const Settings & settings, bool allow_self_aliases)
 {
+    UserDefinedFunctionsVisitor::Data data_user_defined_functions_visitor;
+    UserDefinedFunctionsVisitor(data_user_defined_functions_visitor).visit(query);
+
     CustomizeCountDistinctVisitor::Data data_count_distinct{settings.count_distinct_implementation};
     CustomizeCountDistinctVisitor(data_count_distinct).visit(query);
 

src/Interpreters/UserDefinedFunctionFactory.cpp

@@ -0,0 +1,83 @@
+#include "UserDefinedFunctionFactory.h"
+
+#include <Functions/FunctionFactory.h>
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int FUNCTION_ALREADY_EXISTS;
+    extern const int UNKNOWN_FUNCTION;
+    extern const int CANNOT_DROP_SYSTEM_FUNCTION;
+}
+
+UserDefinedFunctionFactory & UserDefinedFunctionFactory::instance()
+{
+    static UserDefinedFunctionFactory result;
+    return result;
+}
+
+void UserDefinedFunctionFactory::registerFunction(const String & function_name, ASTPtr create_function_query)
+{
+    if (FunctionFactory::instance().hasNameOrAlias(function_name))
+        throw Exception(ErrorCodes::FUNCTION_ALREADY_EXISTS, "The function '{}' already exists", function_name);
+
+    if (AggregateFunctionFactory::instance().hasNameOrAlias(function_name))
+        throw Exception(ErrorCodes::FUNCTION_ALREADY_EXISTS, "The aggregate function '{}' already exists", function_name);
+
+    auto [_, inserted] = function_name_to_create_query.emplace(function_name, std::move(create_function_query));
+    if (!inserted)
+        throw Exception(ErrorCodes::FUNCTION_ALREADY_EXISTS,
+            "The function name '{}' is not unique",
+            function_name);
+}
+
+void UserDefinedFunctionFactory::unregisterFunction(const String & function_name)
+{
+    if (FunctionFactory::instance().hasNameOrAlias(function_name) ||
+        AggregateFunctionFactory::instance().hasNameOrAlias(function_name))
+        throw Exception(ErrorCodes::CANNOT_DROP_SYSTEM_FUNCTION, "Cannot drop system function '{}'", function_name);
+
+    auto it = function_name_to_create_query.find(function_name);
+    if (it == function_name_to_create_query.end())
+        throw Exception(ErrorCodes::UNKNOWN_FUNCTION,
+            "The function name '{}' is not registered",
+            function_name);
+
+    function_name_to_create_query.erase(it);
+}
+
+ASTPtr UserDefinedFunctionFactory::get(const String & function_name) const
+{
+    auto it = function_name_to_create_query.find(function_name);
+    if (it == function_name_to_create_query.end())
+        throw Exception(ErrorCodes::UNKNOWN_FUNCTION,
+            "The function name '{}' is not registered",
+            function_name);
+
+    return it->second;
+}
+
+ASTPtr UserDefinedFunctionFactory::tryGet(const std::string & function_name) const
+{
+    auto it = function_name_to_create_query.find(function_name);
+    if (it == function_name_to_create_query.end())
+        return nullptr;
+
+    return it->second;
+}
+
+std::vector<std::string> UserDefinedFunctionFactory::getAllRegisteredNames() const
+{
+    std::vector<std::string> registered_names;
+    registered_names.reserve(function_name_to_create_query.size());
+
+    for (const auto & [name, _] : function_name_to_create_query)
+        registered_names.emplace_back(name);
+
+    return registered_names;
+}
+
+}

src/Interpreters/UserDefinedFunctionFactory.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <unordered_map>
+
+#include <Common/NamePrompter.h>
+
+#include <Parsers/ASTCreateFunctionQuery.h>
+
+namespace DB
+{
+
+class UserDefinedFunctionFactory : public IHints<1, UserDefinedFunctionFactory>
+{
+public:
+    static UserDefinedFunctionFactory & instance();
+
+    void registerFunction(const String & function_name, ASTPtr create_function_query);
+
+    void unregisterFunction(const String & function_name);
+
+    ASTPtr get(const String & function_name) const;
+
+    ASTPtr tryGet(const String & function_name) const;
+
+    std::vector<String> getAllRegisteredNames() const override;
+
+private:
+
+    std::unordered_map<String, ASTPtr> function_name_to_create_query;
+};
+
+}

src/Interpreters/UserDefinedFunctionsVisitor.cpp

@@ -0,0 +1,99 @@
+#include "UserDefinedFunctionsVisitor.h"
+
+#include <unordered_map>
+#include <stack>
+
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTCreateFunctionQuery.h>
+#include <Parsers/ASTExpressionList.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Interpreters/UserDefinedFunctionFactory.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int UNSUPPORTED_METHOD;
+}
+
+void UserDefinedFunctionsMatcher::visit(ASTPtr & ast, Data &)
+{
+    auto * function = ast->as<ASTFunction>();
+    if (!function)
+        return;
+
+    auto result = tryToReplaceFunction(*function);
+    if (result)
+        ast = result;
+}
+
+bool UserDefinedFunctionsMatcher::needChildVisit(const ASTPtr &, const ASTPtr &)
+{
+    return true;
+}
+
+ASTPtr UserDefinedFunctionsMatcher::tryToReplaceFunction(const ASTFunction & function)
+{
+    auto user_defined_function = UserDefinedFunctionFactory::instance().tryGet(function.name);
+    if (!user_defined_function)
+        return nullptr;
+
+    const auto & function_arguments_list = function.children.at(0)->as<ASTExpressionList>();
+    auto & function_arguments = function_arguments_list->children;
+
+    const auto & create_function_query = user_defined_function->as<ASTCreateFunctionQuery>();
+    auto & function_core_expression = create_function_query->function_core->children.at(0);
+
+    const auto & identifiers_expression_list = function_core_expression->children.at(0)->children.at(0)->as<ASTExpressionList>();
+    const auto & identifiers_raw = identifiers_expression_list->children;
+
+    if (function_arguments.size() != identifiers_raw.size())
+        throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
+            "Function {} expects {} arguments actual arguments {}",
+            create_function_query->function_name,
+            identifiers_raw.size(),
+            function_arguments.size());
+
+    std::unordered_map<std::string, ASTPtr> identifier_name_to_function_argument;
+
+    for (size_t parameter_index = 0; parameter_index < identifiers_raw.size(); ++parameter_index)
+    {
+        const auto & identifier = identifiers_raw[parameter_index]->as<ASTIdentifier>();
+        const auto & function_argument = function_arguments[parameter_index];
+        const auto & identifier_name = identifier->name();
+
+        identifier_name_to_function_argument.emplace(identifier_name, function_argument);
+    }
+
+    auto function_body_to_update = function_core_expression->children.at(1)->clone();
+
+    std::stack<ASTPtr> ast_nodes_to_update;
+    ast_nodes_to_update.push(function_body_to_update);
+
+    while (!ast_nodes_to_update.empty())
+    {
+        auto ast_node_to_update = ast_nodes_to_update.top();
+        ast_nodes_to_update.pop();
+
+        for (auto & child : ast_node_to_update->children)
+        {
+            auto identifier_name_opt = tryGetIdentifierName(child);
+            if (identifier_name_opt)
+            {
+                auto function_argument_it = identifier_name_to_function_argument.find(*identifier_name_opt);
+                assert(function_argument_it != identifier_name_to_function_argument.end());
+
+                child = function_argument_it->second->clone();
+                continue;
+            }
+
+            ast_nodes_to_update.push(child);
+        }
+    }
+
+    return function_body_to_update;
+}
+
+}

src/Interpreters/UserDefinedFunctionsVisitor.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <Interpreters/Aliases.h>
+#include <Interpreters/InDepthNodeVisitor.h>
+
+namespace DB
+{
+
+class ASTFunction;
+
+/** Visits ASTFunction nodes and if it is used defined function replace it with function body.
+  * Example:
+  *
+  * CREATE FUNCTION test_function AS a -> a + 1;
+  *
+  * Before applying visitor:
+  * SELECT test_function(number) FROM system.numbers LIMIT 10;
+  *
+  * After applying visitor:
+  * SELECT number + 1 FROM system.numbers LIMIT 10;
+  */
+class UserDefinedFunctionsMatcher
+{
+public:
+    using Visitor = InDepthNodeVisitor<UserDefinedFunctionsMatcher, true>;
+
+    struct Data
+    {
+    };
+
+    static void visit(ASTPtr & ast, Data & data);
+    static bool needChildVisit(const ASTPtr & node, const ASTPtr & child);
+
+private:
+    static void visit(ASTFunction & func, const Data & data);
+
+    static ASTPtr tryToReplaceFunction(const ASTFunction & function);
+
+};
+
+/// Visits AST nodes and collect their aliases in one map (with links to source nodes).
+using UserDefinedFunctionsVisitor = UserDefinedFunctionsMatcher::Visitor;
+
+}

src/Interpreters/UserDefinedObjectsLoader.cpp

@@ -0,0 +1,164 @@
+#include "UserDefinedObjectsLoader.h"
+
+#include <filesystem>
+
+#include <Common/escapeForFileName.h>
+#include <Common/quoteString.h>
+#include <Common/StringUtils/StringUtils.h>
+
+#include <IO/ReadBufferFromFile.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteBufferFromFile.h>
+#include <IO/WriteHelpers.h>
+
+#include <Interpreters/Context.h>
+#include <Interpreters/InterpreterCreateFunctionQuery.h>
+
+#include <Parsers/parseQuery.h>
+#include <Parsers/ASTCreateFunctionQuery.h>
+#include <Parsers/formatAST.h>
+#include <Parsers/ParserCreateFunctionQuery.h>
+
+#include <Poco/DirectoryIterator.h>
+#include <Poco/Logger.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int OBJECT_ALREADY_STORED_ON_DISK;
+    extern const int OBJECT_WAS_NOT_STORED_ON_DISK;
+}
+
+UserDefinedObjectsLoader & UserDefinedObjectsLoader::instance()
+{
+    static UserDefinedObjectsLoader ret;
+    return ret;
+}
+
+UserDefinedObjectsLoader::UserDefinedObjectsLoader()
+    : log(&Poco::Logger::get("UserDefinedObjectsLoader"))
+{}
+
+void UserDefinedObjectsLoader::loadUserDefinedObject(ContextPtr context, UserDefinedObjectType object_type, const std::string_view & name, const String & path)
+{
+    auto name_ref = StringRef(name.data(), name.size());
+    LOG_DEBUG(log, "Loading user defined object {} from file {}", backQuote(name_ref), path);
+
+    /// There is .sql file with user defined object creation statement.
+    ReadBufferFromFile in(path);
+
+    String object_create_query;
+    readStringUntilEOF(object_create_query, in);
+
+    try
+    {
+        switch (object_type)
+        {
+            case UserDefinedObjectType::Function:
+            {
+                ParserCreateFunctionQuery parser;
+                ASTPtr ast = parseQuery(
+                    parser,
+                    object_create_query.data(),
+                    object_create_query.data() + object_create_query.size(),
+                    "in file " + path,
+                    0,
+                    context->getSettingsRef().max_parser_depth);
+
+                InterpreterCreateFunctionQuery interpreter(ast, context, true /*is internal*/);
+                interpreter.execute();
+            }
+        }
+    }
+    catch (Exception & e)
+    {
+        e.addMessage(fmt::format("while loading user defined objects {} from path {}", backQuote(name_ref), path));
+        throw;
+    }
+}
+
+void UserDefinedObjectsLoader::loadObjects(ContextPtr context)
+{
+    LOG_DEBUG(log, "loading user defined objects");
+
+    String dir_path = context->getPath() + "user_defined/";
+    Poco::DirectoryIterator dir_end;
+    for (Poco::DirectoryIterator it(dir_path); it != dir_end; ++it)
+    {
+        if (it->isLink())
+            continue;
+
+        const auto & file_name = it.name();
+
+        /// For '.svn', '.gitignore' directory and similar.
+        if (file_name.at(0) == '.')
+            continue;
+
+        if (!it->isDirectory() && endsWith(file_name, ".sql"))
+        {
+            std::string_view object_name = file_name;
+            object_name.remove_suffix(strlen(".sql"));
+            object_name.remove_prefix(strlen("function_"));
+            loadUserDefinedObject(context, UserDefinedObjectType::Function, object_name, dir_path + it.name());
+        }
+    }
+}
+
+void UserDefinedObjectsLoader::storeObject(ContextPtr context, UserDefinedObjectType object_type, const String & object_name, const IAST & ast)
+{
+    String dir_path = context->getPath() + "user_defined/";
+    String file_path;
+
+    switch (object_type)
+    {
+        case UserDefinedObjectType::Function:
+        {
+            file_path = dir_path + "function_" + escapeForFileName(object_name) + ".sql";
+        }
+    }
+
+    if (std::filesystem::exists(file_path))
+        throw Exception(ErrorCodes::OBJECT_ALREADY_STORED_ON_DISK, "User defined object {} already stored on disk", backQuote(file_path));
+
+    LOG_DEBUG(log, "Storing object {} to file {}", backQuote(object_name), file_path);
+
+    WriteBufferFromOwnString create_statement_buf;
+    formatAST(ast, create_statement_buf, false);
+    writeChar('\n', create_statement_buf);
+
+    String create_statement = create_statement_buf.str();
+    WriteBufferFromFile out(file_path, create_statement.size(), O_WRONLY | O_CREAT | O_EXCL);
+    writeString(create_statement, out);
+    out.next();
+    if (context->getSettingsRef().fsync_metadata)
+        out.sync();
+    out.close();
+
+    LOG_DEBUG(log, "Stored object {}", backQuote(object_name));
+}
+
+void UserDefinedObjectsLoader::removeObject(ContextPtr context, UserDefinedObjectType object_type, const String & object_name)
+{
+    String dir_path = context->getPath() + "user_defined/";
+    LOG_DEBUG(log, "Removing file for user defined object {} from {}", backQuote(object_name), dir_path);
+
+    std::filesystem::path file_path;
+
+    switch (object_type)
+    {
+        case UserDefinedObjectType::Function:
+        {
+            file_path = dir_path + "function_" + escapeForFileName(object_name) + ".sql";
+        }
+    }
+
+    if (!std::filesystem::exists(file_path))
+        throw Exception(ErrorCodes::OBJECT_WAS_NOT_STORED_ON_DISK, "User defined object {} was not stored on disk", backQuote(file_path.string()));
+
+    std::filesystem::remove(file_path);
+}
+
+}

src/Interpreters/UserDefinedObjectsLoader.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <Interpreters/Context_fwd.h>
+#include <Parsers/IAST.h>
+
+#include <boost/noncopyable.hpp>
+
+
+namespace DB
+{
+
+enum class UserDefinedObjectType
+{
+    Function
+};
+
+class UserDefinedObjectsLoader : private boost::noncopyable
+{
+public:
+    static UserDefinedObjectsLoader & instance();
+    UserDefinedObjectsLoader();
+
+    void loadObjects(ContextPtr context);
+    void storeObject(ContextPtr context, UserDefinedObjectType object_type, const String & object_name, const IAST & ast);
+    void removeObject(ContextPtr context, UserDefinedObjectType object_type, const String & object_name);
+
+private:
+
+    void loadUserDefinedObject(ContextPtr context, UserDefinedObjectType object_type, const std::string_view & object_name, const String & file_path);
+    Poco::Logger * log;
+};
+
+}

src/Interpreters/ya.make

@@ -71,6 +71,7 @@ SRCS(
     InternalTextLogsQueue.cpp
     InterpreterAlterQuery.cpp
     InterpreterCheckQuery.cpp
+    InterpreterCreateFunctionQuery.cpp
     InterpreterCreateQuery.cpp
     InterpreterCreateQuotaQuery.cpp
     InterpreterCreateRoleQuery.cpp
@@ -79,6 +80,7 @@ SRCS(
     InterpreterCreateUserQuery.cpp
     InterpreterDescribeQuery.cpp
     InterpreterDropAccessEntityQuery.cpp
+    InterpreterDropFunctionQuery.cpp
     InterpreterDropQuery.cpp
     InterpreterExistsQuery.cpp
     InterpreterExplainQuery.cpp
@@ -89,6 +91,7 @@ SRCS(
     InterpreterKillQueryQuery.cpp
     InterpreterOptimizeQuery.cpp
     InterpreterRenameQuery.cpp
+    InterpreterSelectIntersectExceptQuery.cpp
     InterpreterSelectQuery.cpp
     InterpreterSelectWithUnionQuery.cpp
     InterpreterSetQuery.cpp
@@ -142,6 +145,7 @@ SRCS(
     RewriteFunctionToSubcolumnVisitor.cpp
     RewriteSumIfFunctionVisitor.cpp
     RowRefs.cpp
+    SelectIntersectExceptQueryVisitor.cpp
     Set.cpp
     SetVariants.cpp
     SortedBlocksWriter.cpp
@@ -157,6 +161,9 @@ SRCS(
     TranslateQualifiedNamesVisitor.cpp
     TreeOptimizer.cpp
     TreeRewriter.cpp
+    UserDefinedFunctionFactory.cpp
+    UserDefinedFunctionsVisitor.cpp
+    UserDefinedObjectsLoader.cpp
     WindowDescription.cpp
     ZooKeeperLog.cpp
     addMissingDefaults.cpp

src/Parsers/ASTCreateFunctionQuery.cpp

@@ -0,0 +1,21 @@
+#include <Common/quoteString.h>
+#include <IO/Operators.h>
+#include <Parsers/ASTCreateFunctionQuery.h>
+
+namespace DB
+{
+
+ASTPtr ASTCreateFunctionQuery::clone() const
+{
+    return std::make_shared<ASTCreateFunctionQuery>(*this);
+}
+
+void ASTCreateFunctionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState & state, IAST::FormatStateStacked frame) const
+{
+    settings.ostr << (settings.hilite ? hilite_keyword : "") << "CREATE FUNCTION " << (settings.hilite ? hilite_none : "");
+    settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(function_name) << (settings.hilite ? hilite_none : "");
+    settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "");
+    function_core->formatImpl(settings, state, frame);
+}
+
+}

src/Parsers/ASTCreateFunctionQuery.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <Parsers/ASTExpressionList.h>
+#include <Parsers/ASTFunction.h>
+
+namespace DB
+{
+
+class ASTCreateFunctionQuery : public IAST
+{
+public:
+    String function_name;
+    ASTPtr function_core;
+
+    String getID(char) const override { return "CreateFunctionQuery"; }
+
+    ASTPtr clone() const override;
+
+    void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override;
+};
+
+}

src/Parsers/ASTDropFunctionQuery.cpp

@@ -0,0 +1,19 @@
+#include <Parsers/ASTDropFunctionQuery.h>
+#include <Common/quoteString.h>
+#include <IO/Operators.h>
+
+namespace DB
+{
+
+ASTPtr ASTDropFunctionQuery::clone() const
+{
+    return std::make_shared<ASTDropFunctionQuery>(*this);
+}
+
+void ASTDropFunctionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState &, IAST::FormatStateStacked) const
+{
+    settings.ostr << (settings.hilite ? hilite_keyword : "") << "DROP FUNCTION " << (settings.hilite ? hilite_none : "");
+    settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(function_name) << (settings.hilite ? hilite_none : "");
+}
+
+}

src/Parsers/ASTDropFunctionQuery.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "IAST.h"
+
+namespace DB
+{
+
+class ASTDropFunctionQuery : public IAST
+{
+public:
+    String function_name;
+
+    String getID(char) const override { return "DropFunctionQuery"; }
+
+    ASTPtr clone() const override;
+
+    void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override;
+};
+
+}

src/Parsers/ASTFunction.cpp

@@ -13,6 +13,7 @@
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTSubquery.h>
 #include <Parsers/ASTWithAlias.h>
+#include <Parsers/queryToString.h>
 
 
 namespace DB
@@ -21,6 +22,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int UNEXPECTED_EXPRESSION;
+    extern const int UNEXPECTED_AST_STRUCTURE;
 }
 
 void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const
@@ -557,4 +559,33 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
     }
 }
 
+String getFunctionName(const IAST * ast)
+{
+    String res;
+    if (tryGetFunctionNameInto(ast, res))
+        return res;
+    throw Exception(ast ? queryToString(*ast) + " is not an function" : "AST node is nullptr", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
+}
+
+std::optional<String> tryGetFunctionName(const IAST * ast)
+{
+    String res;
+    if (tryGetFunctionNameInto(ast, res))
+        return res;
+    return {};
+}
+
+bool tryGetFunctionNameInto(const IAST * ast, String & name)
+{
+    if (ast)
+    {
+        if (const auto * node = ast->as<ASTFunction>())
+        {
+            name = node->name;
+            return true;
+        }
+    }
+    return false;
+}
+
 }

src/Parsers/ASTFunction.h

@@ -71,4 +71,14 @@ std::shared_ptr<ASTFunction> makeASTFunction(const String & name, Args &&... arg
     return function;
 }
 
+/// ASTFunction Helpers: hide casts and semantic.
+
+String getFunctionName(const IAST * ast);
+std::optional<String> tryGetFunctionName(const IAST * ast);
+bool tryGetFunctionNameInto(const IAST * ast, String & name);
+
+inline String getFunctionName(const ASTPtr & ast) { return getFunctionName(ast.get()); }
+inline std::optional<String> tryGetFunctionName(const ASTPtr & ast) { return tryGetFunctionName(ast.get()); }
+inline bool tryGetFunctionNameInto(const ASTPtr & ast, String & name) { return tryGetFunctionNameInto(ast.get(), name); }
+
 }

src/Parsers/ParserCreateFunctionQuery.cpp

@@ -0,0 +1,47 @@
+#include <Parsers/ASTCreateFunctionQuery.h>
+#include <Parsers/ASTExpressionList.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Parsers/CommonParsers.h>
+#include <Parsers/ExpressionElementParsers.h>
+#include <Parsers/ExpressionListParsers.h>
+#include <Parsers/ParserCreateFunctionQuery.h>
+
+namespace DB
+{
+
+bool ParserCreateFunctionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected)
+{
+    ParserKeyword s_create("CREATE");
+    ParserKeyword s_function("FUNCTION");
+    ParserIdentifier function_name_p;
+    ParserKeyword s_as("AS");
+    ParserLambdaExpression lambda_p;
+
+    ASTPtr function_name;
+    ASTPtr function_core;
+
+    if (!s_create.ignore(pos, expected))
+        return false;
+
+    if (!s_function.ignore(pos, expected))
+        return false;
+
+    if (!function_name_p.parse(pos, function_name, expected))
+        return false;
+
+    if (!s_as.ignore(pos, expected))
+        return false;
+
+    if (!lambda_p.parse(pos, function_core, expected))
+        return false;
+
+    auto create_function_query = std::make_shared<ASTCreateFunctionQuery>();
+    node = create_function_query;
+
+    create_function_query->function_name = function_name->as<ASTIdentifier &>().name();
+    create_function_query->function_core = function_core;
+
+    return true;
+}
+
+}

src/Parsers/ParserCreateFunctionQuery.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "IParserBase.h"
+
+namespace DB
+{
+
+/// CREATE FUNCTION test AS x -> x || '1'
+class ParserCreateFunctionQuery : public IParserBase
+{
+protected:
+    const char * getName() const override { return "CREATE FUNCTION query"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+}

src/Parsers/ParserDropFunctionQuery.cpp

@@ -0,0 +1,35 @@
+#include <Parsers/ASTDropFunctionQuery.h>
+#include <Parsers/ASTIdentifier.h>
+#include <Parsers/CommonParsers.h>
+#include <Parsers/ExpressionElementParsers.h>
+#include <Parsers/ParserDropFunctionQuery.h>
+
+namespace DB
+{
+
+bool ParserDropFunctionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected)
+{
+    ParserKeyword s_drop("DROP");
+    ParserKeyword s_function("FUNCTION");
+    ParserIdentifier function_name_p;
+
+    ASTPtr function_name;
+
+    if (!s_drop.ignore(pos, expected))
+        return false;
+
+    if (!s_function.ignore(pos, expected))
+        return false;
+
+    if (!function_name_p.parse(pos, function_name, expected))
+        return false;
+
+    auto drop_function_query = std::make_shared<ASTDropFunctionQuery>();
+    node = drop_function_query;
+
+    drop_function_query->function_name = function_name->as<ASTIdentifier &>().name();
+
+    return true;
+}
+
+}

src/Parsers/ParserDropFunctionQuery.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "IParserBase.h"
+
+namespace DB
+{
+/// DROP FUNCTION function1
+class ParserDropFunctionQuery : public IParserBase
+{
+protected:
+    const char * getName() const override { return "DROP FUNCTION query"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+}

src/Parsers/ParserQuery.cpp

@@ -1,4 +1,5 @@
 #include <Parsers/ParserAlterQuery.h>
+#include <Parsers/ParserCreateFunctionQuery.h>
 #include <Parsers/ParserBackupQuery.h>
 #include <Parsers/ParserCreateQuery.h>
 #include <Parsers/ParserCreateQuotaQuery.h>
@@ -7,6 +8,7 @@
 #include <Parsers/ParserCreateSettingsProfileQuery.h>
 #include <Parsers/ParserCreateUserQuery.h>
 #include <Parsers/ParserDropAccessEntityQuery.h>
+#include <Parsers/ParserDropFunctionQuery.h>
 #include <Parsers/ParserDropQuery.h>
 #include <Parsers/ParserGrantQuery.h>
 #include <Parsers/ParserInsertQuery.h>
@@ -37,6 +39,8 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ParserCreateQuotaQuery create_quota_p;
     ParserCreateRowPolicyQuery create_row_policy_p;
     ParserCreateSettingsProfileQuery create_settings_profile_p;
+    ParserCreateFunctionQuery create_function_p;
+    ParserDropFunctionQuery drop_function_p;
     ParserDropAccessEntityQuery drop_access_entity_p;
     ParserGrantQuery grant_p;
     ParserSetRoleQuery set_role_p;
@@ -54,6 +58,8 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         || create_quota_p.parse(pos, node, expected)
         || create_row_policy_p.parse(pos, node, expected)
         || create_settings_profile_p.parse(pos, node, expected)
+        || create_function_p.parse(pos, node, expected)
+        || drop_function_p.parse(pos, node, expected)
         || drop_access_entity_p.parse(pos, node, expected)
         || grant_p.parse(pos, node, expected)
         || external_ddl_p.parse(pos, node, expected)

src/Parsers/ya.make

@@ -15,6 +15,7 @@ SRCS(
     ASTColumnsMatcher.cpp
     ASTColumnsTransformers.cpp
     ASTConstraintDeclaration.cpp
+    ASTCreateFunctionQuery.cpp
     ASTCreateQuery.cpp
     ASTCreateQuotaQuery.cpp
     ASTCreateRoleQuery.cpp
@@ -25,6 +26,7 @@ SRCS(
     ASTDictionary.cpp
     ASTDictionaryAttributeDeclaration.cpp
     ASTDropAccessEntityQuery.cpp
+    ASTDropFunctionQuery.cpp
     ASTDropQuery.cpp
     ASTExpressionList.cpp
     ASTFunction.cpp
@@ -89,6 +91,7 @@ SRCS(
     ParserAlterQuery.cpp
     ParserCase.cpp
     ParserCheckQuery.cpp
+    ParserCreateFunctionQuery.cpp
     ParserCreateQuery.cpp
     ParserCreateQuotaQuery.cpp
     ParserCreateRoleQuery.cpp
@@ -101,6 +104,7 @@ SRCS(
     ParserDictionary.cpp
     ParserDictionaryAttributeDeclaration.cpp
     ParserDropAccessEntityQuery.cpp
+    ParserDropFunctionQuery.cpp
     ParserDropQuery.cpp
     ParserExplainQuery.cpp
     ParserExternalDDLQuery.cpp

src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp

@@ -10,6 +10,7 @@
 #include <Storages/MergeTree/MergeTreeDataWriter.h>
 #include <Storages/MergeTree/StorageFromMergeTreeDataPart.h>
 #include <DataStreams/TTLBlockInputStream.h>
+#include <DataStreams/TTLCalcInputStream.h>
 #include <DataStreams/DistinctSortedBlockInputStream.h>
 #include <DataStreams/ExpressionBlockInputStream.h>
 #include <DataStreams/MaterializingBlockInputStream.h>
@@ -493,7 +494,6 @@ static void extractMergingAndGatheringColumns(
     const NamesAndTypesList & storage_columns,
     const ExpressionActionsPtr & sorting_key_expr,
     const IndicesDescription & indexes,
-    const ProjectionsDescription & projections,
     const MergeTreeData::MergingParams & merging_params,
     NamesAndTypesList & gathering_columns, Names & gathering_column_names,
     NamesAndTypesList & merging_columns, Names & merging_column_names)
@@ -507,13 +507,6 @@ static void extractMergingAndGatheringColumns(
                   std::inserter(key_columns, key_columns.end()));
     }
 
-    for (const auto & projection : projections)
-    {
-        Names projection_columns_vec = projection.required_columns;
-        std::copy(projection_columns_vec.cbegin(), projection_columns_vec.cend(),
-                  std::inserter(key_columns, key_columns.end()));
-    }
-
     /// Force sign column for Collapsing mode
     if (merging_params.mode == MergeTreeData::MergingParams::Collapsing)
         key_columns.emplace(merging_params.sign_column);
@@ -727,7 +720,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
         storage_columns,
         metadata_snapshot->getSortingKey().expression,
         metadata_snapshot->getSecondaryIndices(),
-        metadata_snapshot->getProjections(),
         merging_params,
         gathering_columns,
         gathering_column_names,
@@ -1288,10 +1280,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
     auto mrk_extension = source_part->index_granularity_info.is_adaptive ? getAdaptiveMrkExtension(new_data_part->getType())
                                                                          : getNonAdaptiveMrkExtension();
     bool need_sync = needSyncPart(source_part->rows_count, source_part->getBytesOnDisk(), *data_settings);
-    bool need_remove_expired_values = false;
+    auto execute_ttl_type = ExecuteTTLType::NONE;
 
-    if (in && shouldExecuteTTL(metadata_snapshot, interpreter->getColumnDependencies(), commands_for_part))
-        need_remove_expired_values = true;
+    if (in)
+        execute_ttl_type = shouldExecuteTTL(metadata_snapshot, interpreter->getColumnDependencies());
 
     /// All columns from part are changed and may be some more that were missing before in part
     /// TODO We can materialize compact part without copying data
@@ -1319,7 +1311,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
             time_of_mutation,
             compression_codec,
             merge_entry,
-            need_remove_expired_values,
+            execute_ttl_type,
             need_sync,
             space_reservation,
             holder,
@@ -1356,7 +1348,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
             return data.cloneAndLoadDataPartOnSameDisk(source_part, "tmp_clone_", future_part.part_info, metadata_snapshot);
         }
 
-        if (need_remove_expired_values)
+        if (execute_ttl_type != ExecuteTTLType::NONE)
             files_to_skip.insert("ttl.txt");
 
         disk->createDirectories(new_part_tmp_path);
@@ -1416,7 +1408,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
                 time_of_mutation,
                 compression_codec,
                 merge_entry,
-                need_remove_expired_values,
+                execute_ttl_type,
                 need_sync,
                 space_reservation,
                 holder,
@@ -1437,7 +1429,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
             }
         }
 
-        finalizeMutatedPart(source_part, new_data_part, need_remove_expired_values, compression_codec);
+        finalizeMutatedPart(source_part, new_data_part, execute_ttl_type, compression_codec);
     }
 
     return new_data_part;
@@ -1984,21 +1976,22 @@ std::set<MergeTreeProjectionPtr> MergeTreeDataMergerMutator::getProjectionsToRec
     return projections_to_recalc;
 }
 
-bool MergeTreeDataMergerMutator::shouldExecuteTTL(
-    const StorageMetadataPtr & metadata_snapshot, const ColumnDependencies & dependencies, const MutationCommands & commands)
+ExecuteTTLType MergeTreeDataMergerMutator::shouldExecuteTTL(const StorageMetadataPtr & metadata_snapshot, const ColumnDependencies & dependencies)
 {
     if (!metadata_snapshot->hasAnyTTL())
-        return false;
+        return ExecuteTTLType::NONE;
 
-    for (const auto & command : commands)
-        if (command.type == MutationCommand::MATERIALIZE_TTL)
-            return true;
+    bool has_ttl_expression = false;
 
     for (const auto & dependency : dependencies)
-        if (dependency.kind == ColumnDependency::TTL_EXPRESSION || dependency.kind == ColumnDependency::TTL_TARGET)
-            return true;
+    {
+        if (dependency.kind == ColumnDependency::TTL_EXPRESSION)
+            has_ttl_expression = true;
 
-    return false;
+        if (dependency.kind == ColumnDependency::TTL_TARGET)
+            return ExecuteTTLType::NORMAL;
+    }
+    return has_ttl_expression ? ExecuteTTLType::RECALCULATE : ExecuteTTLType::NONE;
 }
 
 // 1. get projection pipeline and a sink to write parts
@@ -2172,7 +2165,7 @@ void MergeTreeDataMergerMutator::mutateAllPartColumns(
     time_t time_of_mutation,
     const CompressionCodecPtr & compression_codec,
     MergeListEntry & merge_entry,
-    bool need_remove_expired_values,
+    ExecuteTTLType execute_ttl_type,
     bool need_sync,
     const ReservationPtr & space_reservation,
     TableLockHolder & holder,
@@ -2185,9 +2178,12 @@ void MergeTreeDataMergerMutator::mutateAllPartColumns(
         mutating_stream = std::make_shared<MaterializingBlockInputStream>(
             std::make_shared<ExpressionBlockInputStream>(mutating_stream, data.getPrimaryKeyAndSkipIndicesExpression(metadata_snapshot)));
 
-    if (need_remove_expired_values)
+    if (execute_ttl_type == ExecuteTTLType::NORMAL)
         mutating_stream = std::make_shared<TTLBlockInputStream>(mutating_stream, data, metadata_snapshot, new_data_part, time_of_mutation, true);
 
+    if (execute_ttl_type == ExecuteTTLType::RECALCULATE)
+        mutating_stream = std::make_shared<TTLCalcInputStream>(mutating_stream, data, metadata_snapshot, new_data_part, time_of_mutation, true);
+
     IMergeTreeDataPart::MinMaxIndex minmax_idx;
 
     MergedBlockOutputStream out{
@@ -2229,7 +2225,7 @@ void MergeTreeDataMergerMutator::mutateSomePartColumns(
     time_t time_of_mutation,
     const CompressionCodecPtr & compression_codec,
     MergeListEntry & merge_entry,
-    bool need_remove_expired_values,
+    ExecuteTTLType execute_ttl_type,
     bool need_sync,
     const ReservationPtr & space_reservation,
     TableLockHolder & holder,
@@ -2238,9 +2234,12 @@ void MergeTreeDataMergerMutator::mutateSomePartColumns(
     if (mutating_stream == nullptr)
         throw Exception("Cannot mutate part columns with uninitialized mutations stream. It's a bug", ErrorCodes::LOGICAL_ERROR);
 
-    if (need_remove_expired_values)
+    if (execute_ttl_type == ExecuteTTLType::NORMAL)
         mutating_stream = std::make_shared<TTLBlockInputStream>(mutating_stream, data, metadata_snapshot, new_data_part, time_of_mutation, true);
 
+    if (execute_ttl_type == ExecuteTTLType::RECALCULATE)
+        mutating_stream = std::make_shared<TTLCalcInputStream>(mutating_stream, data, metadata_snapshot, new_data_part, time_of_mutation, true);
+
     IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets;
     MergedColumnOnlyOutputStream out(
         new_data_part,
@@ -2279,7 +2278,7 @@ void MergeTreeDataMergerMutator::mutateSomePartColumns(
 void MergeTreeDataMergerMutator::finalizeMutatedPart(
     const MergeTreeDataPartPtr & source_part,
     MergeTreeData::MutableDataPartPtr new_data_part,
-    bool need_remove_expired_values,
+    ExecuteTTLType execute_ttl_type,
     const CompressionCodecPtr & codec)
 {
     auto disk = new_data_part->volume->getDisk();
@@ -2293,7 +2292,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart(
         new_data_part->checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_hash = out_hashing.getHash();
     }
 
-    if (need_remove_expired_values)
+    if (execute_ttl_type != ExecuteTTLType::NONE)
     {
         /// Write a file with ttl infos in json format.
         auto out_ttl = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "ttl.txt", 4096);

src/Storages/MergeTree/MergeTreeDataMergerMutator.h

@@ -23,6 +23,13 @@ enum class SelectPartsDecision
     NOTHING_TO_MERGE = 2,
 };
 
+enum class ExecuteTTLType
+{
+    NONE = 0,
+    NORMAL = 1,
+    RECALCULATE= 2,
+};
+
 /// Auxiliary struct holding metainformation for the future merged or mutated part.
 struct FutureMergedMutatedPart
 {
@@ -200,8 +207,7 @@ private:
         const ProjectionsDescription & all_projections,
         const MutationCommands & commands_for_removes);
 
-    static bool shouldExecuteTTL(
-        const StorageMetadataPtr & metadata_snapshot, const ColumnDependencies & dependencies, const MutationCommands & commands);
+    static ExecuteTTLType shouldExecuteTTL(const StorageMetadataPtr & metadata_snapshot, const ColumnDependencies & dependencies);
 
     /// Return set of indices which should be recalculated during mutation also
     /// wraps input stream into additional expression stream
@@ -242,7 +248,7 @@ private:
         time_t time_of_mutation,
         const CompressionCodecPtr & compression_codec,
         MergeListEntry & merge_entry,
-        bool need_remove_expired_values,
+        ExecuteTTLType execute_ttl_type,
         bool need_sync,
         const ReservationPtr & space_reservation,
         TableLockHolder & holder,
@@ -260,7 +266,7 @@ private:
         time_t time_of_mutation,
         const CompressionCodecPtr & compression_codec,
         MergeListEntry & merge_entry,
-        bool need_remove_expired_values,
+        ExecuteTTLType execute_ttl_type,
         bool need_sync,
         const ReservationPtr & space_reservation,
         TableLockHolder & holder,
@@ -271,7 +277,7 @@ private:
     static void finalizeMutatedPart(
         const MergeTreeDataPartPtr & source_part,
         MergeTreeData::MutableDataPartPtr new_data_part,
-        bool need_remove_expired_values,
+        ExecuteTTLType execute_ttl_type,
         const CompressionCodecPtr & codec);
 
 public :

src/Storages/MergeTree/MergeTreeSettings.h

@@ -117,6 +117,7 @@ struct Settings;
     M(Int64, merge_with_ttl_timeout, 3600 * 4, "Minimal time in seconds, when merge with delete TTL can be repeated.", 0) \
     M(Int64, merge_with_recompression_ttl_timeout, 3600 * 4, "Minimal time in seconds, when merge with recompression TTL can be repeated.", 0) \
     M(Bool, ttl_only_drop_parts, false, "Only drop altogether the expired parts and not partially prune them.", 0) \
+    M(Bool, materialize_ttl_recalculate_only, false, "Only recalculate ttl info when MATERIALIZE TTL", 0) \
     M(Bool, write_final_mark, true, "Write final mark after end of column (0 - disabled, do nothing if index_granularity_bytes=0)", 0) \
     M(Bool, enable_mixed_granularity_parts, true, "Enable parts with adaptive and non adaptive granularity", 0) \
     M(MaxThreads, max_part_loading_threads, 0, "The number of threads to load data parts at startup.", 0) \

src/Storages/MergeTree/StorageFromMergeTreeDataPart.h

@@ -73,6 +73,11 @@ public:
         return storage.getPartitionIDFromQuery(ast, context);
     }
 
+    bool materializeTTLRecalculateOnly() const
+    {
+        return parts.front()->storage.getSettings()->materialize_ttl_recalculate_only;
+    }
+
 protected:
     /// Used in part mutation.
     StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_)

src/Storages/StorageInMemoryMetadata.cpp

@@ -214,7 +214,7 @@ bool StorageInMemoryMetadata::hasAnyGroupByTTL() const
     return !table_ttl.group_by_ttl.empty();
 }
 
-ColumnDependencies StorageInMemoryMetadata::getColumnDependencies(const NameSet & updated_columns) const
+ColumnDependencies StorageInMemoryMetadata::getColumnDependencies(const NameSet & updated_columns, bool include_ttl_target) const
 {
     if (updated_columns.empty())
         return {};
@@ -250,7 +250,7 @@ ColumnDependencies StorageInMemoryMetadata::getColumnDependencies(const NameSet
     if (hasRowsTTL())
     {
         auto rows_expression = getRowsTTL().expression;
-        if (add_dependent_columns(rows_expression, required_ttl_columns))
+        if (add_dependent_columns(rows_expression, required_ttl_columns) && include_ttl_target)
         {
             /// Filter all columns, if rows TTL expression have to be recalculated.
             for (const auto & column : getColumns().getAllPhysical())
@@ -263,13 +263,15 @@ ColumnDependencies StorageInMemoryMetadata::getColumnDependencies(const NameSet
 
     for (const auto & [name, entry] : getColumnTTLs())
     {
-        if (add_dependent_columns(entry.expression, required_ttl_columns))
+        if (add_dependent_columns(entry.expression, required_ttl_columns) && include_ttl_target)
             updated_ttl_columns.insert(name);
     }
 
     for (const auto & entry : getMoveTTLs())
         add_dependent_columns(entry.expression, required_ttl_columns);
 
+    //TODO what about rows_where_ttl and group_by_ttl ??
+
     for (const auto & column : indices_columns)
         res.emplace(column, ColumnDependency::SKIP_INDEX);
     for (const auto & column : projections_columns)

src/Storages/StorageInMemoryMetadata.h

@@ -143,7 +143,7 @@ struct StorageInMemoryMetadata
 
     /// Returns columns, which will be needed to calculate dependencies (skip
     /// indices, TTL expressions) if we update @updated_columns set of columns.
-    ColumnDependencies getColumnDependencies(const NameSet & updated_columns) const;
+    ColumnDependencies getColumnDependencies(const NameSet & updated_columns, bool include_ttl_target) const;
 
     /// Block with ordinary + materialized columns.
     Block getSampleBlock() const;

src/Storages/System/StorageSystemFunctions.cpp

@@ -1,26 +1,38 @@
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
+#include <Parsers/queryToString.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/IFunction.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/UserDefinedFunctionFactory.h>
 #include <Storages/System/StorageSystemFunctions.h>
 
-
 namespace DB
 {
 namespace
 {
     template <typename Factory>
-    void fillRow(MutableColumns & res_columns, const String & name, UInt64 is_aggregate, const Factory & f)
+    void fillRow(MutableColumns & res_columns, const String & name, UInt64 is_aggregate, const String & create_query, const Factory & f)
     {
         res_columns[0]->insert(name);
         res_columns[1]->insert(is_aggregate);
-        res_columns[2]->insert(f.isCaseInsensitive(name));
-        if (f.isAlias(name))
-            res_columns[3]->insert(f.aliasTo(name));
-        else
+
+        if constexpr (std::is_same_v<Factory, UserDefinedFunctionFactory>)
+        {
+            res_columns[2]->insert(false);
             res_columns[3]->insertDefault();
+        }
+        else
+        {
+            res_columns[2]->insert(f.isCaseInsensitive(name));
+            if (f.isAlias(name))
+                res_columns[3]->insert(f.aliasTo(name));
+            else
+                res_columns[3]->insertDefault();
+        }
+
+        res_columns[4]->insert(create_query);
     }
 }
 
@@ -31,6 +43,7 @@ NamesAndTypesList StorageSystemFunctions::getNamesAndTypes()
         {"is_aggregate", std::make_shared<DataTypeUInt8>()},
         {"case_insensitive", std::make_shared<DataTypeUInt8>()},
         {"alias_to", std::make_shared<DataTypeString>()},
+        {"create_query", std::make_shared<DataTypeString>()}
     };
 }
 
@@ -40,14 +53,22 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr,
     const auto & function_names = functions_factory.getAllRegisteredNames();
     for (const auto & function_name : function_names)
     {
-        fillRow(res_columns, function_name, UInt64(0), functions_factory);
+        fillRow(res_columns, function_name, UInt64(0), "", functions_factory);
     }
 
     const auto & aggregate_functions_factory = AggregateFunctionFactory::instance();
     const auto & aggregate_function_names = aggregate_functions_factory.getAllRegisteredNames();
     for (const auto & function_name : aggregate_function_names)
     {
-        fillRow(res_columns, function_name, UInt64(1), aggregate_functions_factory);
+        fillRow(res_columns, function_name, UInt64(1), "", aggregate_functions_factory);
+    }
+
+    const auto & user_defined_functions_factory = UserDefinedFunctionFactory::instance();
+    const auto & user_defined_functions_names = user_defined_functions_factory.getAllRegisteredNames();
+    for (const auto & function_name : user_defined_functions_names)
+    {
+        auto create_query = queryToString(user_defined_functions_factory.get(function_name));
+        fillRow(res_columns, function_name, UInt64(0), create_query, user_defined_functions_factory);
     }
 }
 }

src/Storages/examples/CMakeLists.txt

@@ -22,4 +22,3 @@ target_link_libraries (transform_part_zk_nodes
         dbms
         string_utils
 )
-

src/Storages/fuzzers/CMakeLists.txt

@@ -1,11 +1,7 @@
 
-add_executable (mergetree_checksum_fuzzer
-    mergetree_checksum_fuzzer.cpp
-    "${ClickHouse_SOURCE_DIR}/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp"
-    "${ClickHouse_SOURCE_DIR}/src/Compression/CompressedReadBuffer.cpp"
-    "${ClickHouse_SOURCE_DIR}/src/Compression/CompressedWriteBuffer.cpp"
-)
-target_link_libraries (mergetree_checksum_fuzzer PRIVATE clickhouse_common_io fuzz_compression ${LIB_FUZZING_ENGINE})
+add_executable (mergetree_checksum_fuzzer mergetree_checksum_fuzzer.cpp)
+# Look at comment around fuzz_compression target declaration
+target_link_libraries (mergetree_checksum_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE})
 
 add_executable (columns_description_fuzzer columns_description_fuzzer.cpp)
 target_link_libraries (columns_description_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE})

tests/integration/README.md

@@ -38,7 +38,7 @@ sudo -H pip install \
     pytest \
     pytest-timeout \
     redis \
-    tzlocal \
+    tzlocal==2.1 \
     urllib3 \
     requests-kerberos \
     dict2xml \

tests/integration/test_access_for_functions/test.py

@@ -0,0 +1,39 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+instance = cluster.add_instance('instance')
+
+
+@pytest.fixture(scope="module", autouse=True)
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+def test_access_rights_for_funtion():
+    create_function_query = "CREATE FUNCTION MySum AS (a, b) -> a + b"
+
+    instance.query("CREATE USER A")
+    instance.query("CREATE USER B")
+    assert "it's necessary to have grant CREATE FUNCTION ON *.*" in instance.query_and_get_error(create_function_query, user = 'A')
+
+    instance.query("GRANT CREATE FUNCTION on *.* TO A")
+
+    instance.query(create_function_query, user = 'A')
+    assert instance.query("SELECT MySum(1, 2)") == "3\n"
+
+    assert "it's necessary to have grant DROP FUNCTION ON *.*" in instance.query_and_get_error("DROP FUNCTION MySum", user = 'B')
+
+    instance.query("GRANT DROP FUNCTION ON *.* TO B")
+    instance.query("DROP FUNCTION MySum", user = 'B')
+    assert "Unknown function MySum" in instance.query_and_get_error("SELECT MySum(1, 2)")
+
+    instance.query("REVOKE CREATE FUNCTION ON *.* FROM A")
+    assert "it's necessary to have grant CREATE FUNCTION ON *.*" in instance.query_and_get_error(create_function_query, user = 'A')
+
+    instance.query("DROP USER IF EXISTS A")
+    instance.query("DROP USER IF EXISTS B")

tests/integration/test_user_defined_object_persistence/test.py

@@ -0,0 +1,39 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+instance = cluster.add_instance('instance', stay_alive=True)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def test_persistence():
+    create_function_query1 = "CREATE FUNCTION MySum1 AS (a, b) -> a + b"
+    create_function_query2 = "CREATE FUNCTION MySum2 AS (a, b) -> MySum1(a, b) + b"
+
+    instance.query(create_function_query1)
+    instance.query(create_function_query2)
+    
+    assert instance.query("SELECT MySum1(1,2)") == "3\n"
+    assert instance.query("SELECT MySum2(1,2)") == "5\n"
+
+    instance.restart_clickhouse()
+
+    assert instance.query("SELECT MySum1(1,2)") == "3\n"
+    assert instance.query("SELECT MySum2(1,2)") == "5\n"
+
+    instance.query("DROP FUNCTION MySum2")
+    instance.query("DROP FUNCTION MySum1")
+
+    instance.restart_clickhouse()
+
+    assert "Unknown function MySum1" in instance.query_and_get_error("SELECT MySum1(1, 2)")
+    assert "Unknown function MySum2" in instance.query_and_get_error("SELECT MySum2(1, 2)")

tests/queries/0_stateless/01070_modify_ttl_recalc_only.reference

@@ -0,0 +1,68 @@
+2000-10-10	1
+2000-10-10	2
+2100-10-10	3
+2100-10-10	4
+2000-10-11 00:00:00	2000-10-11 00:00:00
+2000-10-11 00:00:00	2000-10-11 00:00:00
+2100-10-11 00:00:00	2100-10-11 00:00:00
+2100-10-11 00:00:00	2100-10-11 00:00:00
+2100-10-10	3
+2100-10-10	4
+=============
+1	a
+2	b
+3	c
+4	d
+2000-01-01 00:00:00	2100-01-01 00:00:00
+1	a
+3	c
+=============
+1	a
+3	c
+2000-01-01 00:00:00	2000-01-01 00:00:00
+=============
+1	a
+2	b
+3	c
+4	d
+1	a
+2	
+3	c
+4	
+=============
+1	a
+2	
+3	c
+4	
+1	
+2	
+3	
+4	
+=============
+1	a
+2	b
+3	c
+4	d
+2000-01-01 00:00:00	2100-01-01 00:00:00
+1	a
+2	b
+4	d
+=============
+1	a
+2	b
+4	d
+1	
+2	
+4	d
+=============
+1	a	aa
+2	b	bb
+3	c	cc
+4	d	dd
+1	a	
+2	b	bb
+3		cc
+4	d	
+1
+=============
+0

tests/queries/0_stateless/01070_modify_ttl_recalc_only.sql

@@ -0,0 +1,107 @@
+set mutations_sync = 2;
+
+drop table if exists ttl;
+
+create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d)
+SETTINGS max_number_of_merges_with_ttl_in_pool=0,materialize_ttl_recalculate_only=true;
+
+insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1);
+insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2);
+insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3);
+insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4);
+
+
+alter table ttl modify ttl d + interval 1 day;
+select * from ttl order by a;
+select delete_ttl_info_min, delete_ttl_info_max  from system.parts where database = currentDatabase() and table = 'ttl' and active > 0 order by name asc;
+optimize table ttl final;
+select * from ttl order by a;
+select '=============';
+
+drop table if exists ttl;
+
+create table ttl (i Int, s String) engine = MergeTree order by i
+SETTINGS max_number_of_merges_with_ttl_in_pool=0,materialize_ttl_recalculate_only=true;
+
+insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
+
+alter table ttl modify ttl i % 2 = 0 ? toDate('2000-01-01') : toDate('2100-01-01');
+select * from ttl order by i;
+select delete_ttl_info_min, delete_ttl_info_max  from system.parts where database = currentDatabase() and table = 'ttl' and active > 0;
+optimize table ttl final;
+select * from ttl order by i;
+select '=============';
+
+alter table ttl modify ttl toDate('2000-01-01');
+select * from ttl order by i;
+select delete_ttl_info_min, delete_ttl_info_max  from system.parts where database = currentDatabase() and table = 'ttl' and active > 0;
+optimize table ttl final;
+select * from ttl order by i;
+select '=============';
+
+drop table if exists ttl;
+
+create table ttl (i Int, s String) engine = MergeTree order by i
+SETTINGS max_number_of_merges_with_ttl_in_pool=0,materialize_ttl_recalculate_only=true;
+
+insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
+
+alter table ttl modify column s String ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
+select * from ttl order by i;
+optimize table ttl final;
+select * from ttl order by i;
+select '=============';
+
+alter table ttl modify column s String ttl toDate('2000-01-01');
+select * from ttl order by i;
+optimize table ttl final;
+select * from ttl order by i;
+select '=============';
+
+drop table if exists ttl;
+
+create table ttl (d Date, i Int, s String) engine = MergeTree order by i
+SETTINGS max_number_of_merges_with_ttl_in_pool=0,materialize_ttl_recalculate_only=true;
+
+insert into ttl values (toDate('2000-01-02'), 1, 'a') (toDate('2000-01-03'), 2, 'b') (toDate('2080-01-01'), 3, 'c') (toDate('2080-01-03'), 4, 'd');
+
+alter table ttl modify ttl i % 3 = 0 ? toDate('2000-01-01') : toDate('2100-01-01');
+select i, s from ttl order by i;
+select delete_ttl_info_min, delete_ttl_info_max  from system.parts where database = currentDatabase() and table = 'ttl' and active > 0;
+optimize table ttl final;
+select i, s from ttl order by i;
+select '=============';
+
+alter table ttl modify column s String ttl d + interval 1 month;
+select i, s from ttl order by i;
+optimize table ttl final;
+select i, s from ttl order by i;
+select '=============';
+
+drop table if exists ttl;
+
+create table ttl (i Int, s String, t String) engine = MergeTree order by i
+SETTINGS max_number_of_merges_with_ttl_in_pool=0,materialize_ttl_recalculate_only=true;
+
+insert into ttl values (1, 'a', 'aa') (2, 'b', 'bb') (3, 'c', 'cc') (4, 'd', 'dd');
+
+alter table ttl modify column s String ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01'),
+                modify column t String ttl i % 3 = 1 ? today() - 10 : toDate('2100-01-01');
+
+select i, s, t from ttl order by i;
+optimize table ttl final;
+select i, s, t from ttl order by i;
+-- MATERIALIZE TTL ran only once
+select count() from system.mutations where database = currentDatabase() and table = 'ttl' and is_done;
+select '=============';
+
+drop table if exists ttl;
+
+-- Nothing changed, don't run mutation
+create table ttl (i Int, s String ttl toDate('2000-01-02')) engine = MergeTree order by i
+SETTINGS max_number_of_merges_with_ttl_in_pool=0,materialize_ttl_recalculate_only=true;
+
+alter table ttl modify column s String ttl toDate('2000-01-02');
+select count() from system.mutations where database = currentDatabase() and table = 'ttl' and is_done;
+
+drop table if exists ttl;

tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.reference

@@ -3,6 +3,11 @@ distributed_group_by_no_merge
 1
 optimize_skip_unused_shards
 1
-optimize_skip_unused_shards lack of WHERE
+optimize_skip_unused_shards lack of WHERE (optimize_distributed_group_by_sharding_key=0)
+0
+1
+optimize_skip_unused_shards lack of WHERE (optimize_distributed_group_by_sharding_key=1)
+0
+1
 0
 1

tests/queries/0_stateless/01213_optimize_skip_unused_shards_DISTINCT.sql

@@ -12,8 +12,15 @@ SELECT DISTINCT id FROM dist_01213 WHERE id = 1 SETTINGS distributed_group_by_no
 SELECT 'optimize_skip_unused_shards';
 SELECT DISTINCT id FROM dist_01213 WHERE id = 1 SETTINGS optimize_skip_unused_shards=1;
 -- check that querying all shards is ok
-SELECT 'optimize_skip_unused_shards lack of WHERE';
-SELECT DISTINCT id FROM dist_01213 SETTINGS optimize_skip_unused_shards=1;
+SELECT 'optimize_skip_unused_shards lack of WHERE (optimize_distributed_group_by_sharding_key=0)';
+SELECT DISTINCT id FROM dist_01213 SETTINGS optimize_skip_unused_shards=1, optimize_distributed_group_by_sharding_key=0;
+-- with optimize_distributed_group_by_sharding_key=1 there will be 4 rows,
+-- since DISTINCT will be done on each shard separatelly, and initiator will
+-- not do anything (since we use optimize_skip_unused_shards=1 that must
+-- guarantee that the data had been INSERTed according to sharding key,
+-- which is not our case, since we use one local table).
+SELECT 'optimize_skip_unused_shards lack of WHERE (optimize_distributed_group_by_sharding_key=1)';
+SELECT DISTINCT id FROM dist_01213 SETTINGS optimize_skip_unused_shards=1, optimize_distributed_group_by_sharding_key=1;
 
 DROP TABLE local_01213;
 DROP TABLE dist_01213;