Merge branch 'master' into numbers-schema-inference

.github/workflows/cherry_pick.yml

@@ -8,7 +8,7 @@ concurrency:
   group: cherry-pick
 on: # yamllint disable-line rule:truthy
   schedule:
-    - cron: '0 */3 * * *'
+    - cron: '0 * * * *'
   workflow_dispatch:
 
 jobs:

.github/workflows/master.yml

@@ -2902,7 +2902,7 @@ jobs:
 #############################################################################################
 #################################### PERFORMANCE TESTS ######################################
 #############################################################################################
-  PerformanceComparison0:
+  PerformanceComparisonX86-0:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -2940,7 +2940,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison1:
+  PerformanceComparisonX86-1:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -2978,7 +2978,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison2:
+  PerformanceComparisonX86-2:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3016,7 +3016,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison3:
+  PerformanceComparisonX86-3:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3096,10 +3096,10 @@ jobs:
       - IntegrationTestsTsan1
       - IntegrationTestsTsan2
       - IntegrationTestsTsan3
-      - PerformanceComparison0
-      - PerformanceComparison1
-      - PerformanceComparison2
-      - PerformanceComparison3
+      - PerformanceComparisonX86-0
+      - PerformanceComparisonX86-1
+      - PerformanceComparisonX86-2
+      - PerformanceComparisonX86-3
       - CompatibilityCheck
       - ASTFuzzerTestDebug
       - ASTFuzzerTestAsan

.github/workflows/pull_request.yml

@@ -3118,7 +3118,7 @@ jobs:
 #############################################################################################
 #################################### PERFORMANCE TESTS ######################################
 #############################################################################################
-  PerformanceComparison0:
+  PerformanceComparisonX86-0:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3156,7 +3156,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison1:
+  PerformanceComparisonX86-1:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3194,7 +3194,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison2:
+  PerformanceComparisonX86-2:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3232,7 +3232,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison3:
+  PerformanceComparisonX86-3:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3270,7 +3270,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch0:
+  PerformanceComparisonAarch-0:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3308,7 +3308,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch1:
+  PerformanceComparisonAarch-1:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3346,7 +3346,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch2:
+  PerformanceComparisonAarch-2:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3384,7 +3384,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch3:
+  PerformanceComparisonAarch-3:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3481,14 +3481,14 @@ jobs:
       - IntegrationTestsTsan1
       - IntegrationTestsTsan2
       - IntegrationTestsTsan3
-      - PerformanceComparison0
-      - PerformanceComparison1
-      - PerformanceComparison2
-      - PerformanceComparison3
-      - PerformanceComparisonAarch0
-      - PerformanceComparisonAarch1
-      - PerformanceComparisonAarch2
-      - PerformanceComparisonAarch3
+      - PerformanceComparisonX86-0
+      - PerformanceComparisonX86-1
+      - PerformanceComparisonX86-2
+      - PerformanceComparisonX86-3
+      - PerformanceComparisonAarch-0
+      - PerformanceComparisonAarch-1
+      - PerformanceComparisonAarch-2
+      - PerformanceComparisonAarch-3
       - UnitTestsAsan
       - UnitTestsTsan
       - UnitTestsMsan

CMakeLists.txt

@@ -74,18 +74,13 @@ message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
 string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
 
 option(USE_STATIC_LIBRARIES "Disable to use shared libraries" ON)
+# DEVELOPER ONLY.
+# Faster linking if turned on.
+option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files" OFF)
+option(CLICKHOUSE_SPLIT_BINARY "Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled" OFF)
 
-if (NOT USE_STATIC_LIBRARIES)
-    # DEVELOPER ONLY.
-    # Faster linking if turned on.
-    option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files")
-
-    option(CLICKHOUSE_SPLIT_BINARY
-        "Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled")
-endif ()
-
-if (USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
-    message(FATAL_ERROR "Defining SPLIT_SHARED_LIBRARIES=1 without USE_STATIC_LIBRARIES=0 has no effect.")
+if (USE_STATIC_LIBRARIES AND (SPLIT_SHARED_LIBRARIES OR CLICKHOUSE_SPLIT_BINARY))
+    message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 or CLICKHOUSE_SPLIT_BINARY=1 must not be used together with USE_STATIC_LIBRARIES=1")
 endif()
 
 if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
@@ -335,6 +330,22 @@ if (COMPILER_GCC OR COMPILER_CLANG)
     set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
 endif ()
 
+if (ARCH_AMD64)
+    # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
+    # which makes benchmark results more stable.
+    set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
+    if (COMPILER_GCC)
+        # gcc is in assembler, need to add "-Wa," prefix
+        set(BRANCHES_WITHIN_32B_BOUNDARIES "-Wa,${BRANCHES_WITHIN_32B_BOUNDARIES}")
+    endif()
+
+    include(CheckCXXCompilerFlag)
+    check_cxx_compiler_flag("${BRANCHES_WITHIN_32B_BOUNDARIES}" HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
+    if (HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
+        set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
+    endif()
+endif()
+
 if (COMPILER_GCC)
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines")
 endif ()

base/base/JSON.cpp

@@ -12,7 +12,14 @@
 #define JSON_MAX_DEPTH 100
 
 
+#ifdef __clang__
+#  pragma clang diagnostic push
+#  pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec"
+#endif
 POCO_IMPLEMENT_EXCEPTION(JSONException, Poco::Exception, "JSONException") // NOLINT(cert-err60-cpp, modernize-use-noexcept, hicpp-use-noexcept)
+#ifdef __clang__
+#  pragma clang diagnostic pop
+#endif
 
 
 /// Прочитать беззнаковое целое в простом формате из не-0-terminated строки.

base/base/JSON.h

@@ -38,8 +38,14 @@
   */
 
 
+#ifdef __clang__
+#  pragma clang diagnostic push
+#  pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec"
+#endif
 POCO_DECLARE_EXCEPTION(Foundation_API, JSONException, Poco::Exception)
-
+#ifdef __clang__
+#  pragma clang diagnostic pop
+#endif
 
 class JSON
 {

base/base/defines.h

@@ -93,7 +93,6 @@
 #    define NO_SANITIZE_ADDRESS __attribute__((__no_sanitize__("address")))
 #    define NO_SANITIZE_THREAD __attribute__((__no_sanitize__("thread")))
 #    define ALWAYS_INLINE_NO_SANITIZE_UNDEFINED __attribute__((__always_inline__, __no_sanitize__("undefined")))
-#    define DISABLE_SANITIZER_INSTRUMENTATION __attribute__((disable_sanitizer_instrumentation))
 #else  /// It does not work in GCC. GCC 7 cannot recognize this attribute and GCC 8 simply ignores it.
 #    define NO_SANITIZE_UNDEFINED
 #    define NO_SANITIZE_ADDRESS
@@ -101,6 +100,13 @@
 #    define ALWAYS_INLINE_NO_SANITIZE_UNDEFINED ALWAYS_INLINE
 #endif
 
+#if defined(__clang__) && defined(__clang_major__) && __clang_major__ >= 14
+#    define DISABLE_SANITIZER_INSTRUMENTATION __attribute__((disable_sanitizer_instrumentation))
+#else
+#    define DISABLE_SANITIZER_INSTRUMENTATION
+#endif
+
+
 #if !__has_include(<sanitizer/asan_interface.h>) || !defined(ADDRESS_SANITIZER)
 #   define ASAN_UNPOISON_MEMORY_REGION(a, b)
 #   define ASAN_POISON_MEMORY_REGION(a, b)

base/glibc-compatibility/musl/getauxval.c

@@ -1,68 +1,176 @@
-#include <sys/auxv.h>
 #include "atomic.h"
-#include <unistd.h> // __environ
+#include <sys/auxv.h>
+#include <fcntl.h> // open
+#include <sys/stat.h> // O_RDONLY
+#include <unistd.h> // read, close
+#include <stdlib.h> // ssize_t
+#include <stdio.h> // perror, fprintf
+#include <link.h> // ElfW
 #include <errno.h>
 
-// We don't have libc struct available here. Compute aux vector manually.
-static unsigned long * __auxv = NULL;
-static unsigned long __auxv_secure = 0;
+#define ARRAY_SIZE(a) sizeof((a))/sizeof((a[0]))
 
+// We don't have libc struct available here.
+// Compute aux vector manually (from /proc/self/auxv).
+//
+// Right now there is only 51 AT_* constants,
+// so 64 should be enough until this implementation will be replaced with musl.
+static unsigned long __auxv_procfs[64];
+static unsigned long __auxv_secure = 0;
+// Common
+static unsigned long * __auxv_environ = NULL;
+
+static void * volatile getauxval_func;
+
+static unsigned long  __auxv_init_environ(unsigned long type);
+
+//
+// auxv from procfs interface
+//
+ssize_t __retry_read(int fd, void * buf, size_t count)
+{
+    for (;;)
+    {
+        ssize_t ret = read(fd, buf, count);
+        if (ret == -1)
+        {
+            if (errno == EINTR)
+            {
+                continue;
+            }
+            perror("Cannot read /proc/self/auxv");
+            abort();
+        }
+        return ret;
+    }
+}
+unsigned long __getauxval_procfs(unsigned long type)
+{
+    if (type == AT_SECURE)
+    {
+        return __auxv_secure;
+    }
+
+    if (type >= ARRAY_SIZE(__auxv_procfs))
+    {
+        errno = ENOENT;
+        return 0;
+    }
+
+    return __auxv_procfs[type];
+}
+static unsigned long __auxv_init_procfs(unsigned long type)
+{
+    // For debugging:
+    // - od -t dL /proc/self/auxv
+    // - LD_SHOW_AUX= ls
+    int fd = open("/proc/self/auxv", O_RDONLY);
+    // It is possible in case of:
+    // - no procfs mounted
+    // - on android you are not able to read it unless running from shell or debugging
+    // - some other issues
+    if (fd == -1)
+    {
+        // Fallback to environ.
+        a_cas_p(&getauxval_func, (void *)__auxv_init_procfs, (void *)__auxv_init_environ);
+        return __auxv_init_environ(type);
+    }
+
+    ElfW(auxv_t) aux;
+
+    /// NOTE: sizeof(aux) is very small (less then PAGE_SIZE), so partial read should not be possible.
+    _Static_assert(sizeof(aux) < 4096, "Unexpected sizeof(aux)");
+    while (__retry_read(fd, &aux, sizeof(aux)) == sizeof(aux))
+    {
+        if (aux.a_type >= ARRAY_SIZE(__auxv_procfs))
+        {
+            fprintf(stderr, "AT_* is out of range: %li (maximum allowed is %zu)\n", aux.a_type, ARRAY_SIZE(__auxv_procfs));
+            abort();
+        }
+        if (__auxv_procfs[aux.a_type])
+        {
+            fprintf(stderr, "AUXV already has value (%zu)\n", __auxv_procfs[aux.a_type]);
+            abort();
+        }
+        __auxv_procfs[aux.a_type] = aux.a_un.a_val;
+    }
+    close(fd);
+
+    __auxv_secure = __getauxval_procfs(AT_SECURE);
+
+    // Now we've initialized __auxv_procfs, next time getauxval() will only call __get_auxval().
+    a_cas_p(&getauxval_func, (void *)__auxv_init_procfs, (void *)__getauxval_procfs);
+
+    return __getauxval_procfs(type);
+}
+
+//
+// auxv from environ interface
+//
+// NOTE: environ available only after static initializers,
+// so you cannot rely on this if you need getauxval() before.
+//
+// Good example of such user is sanitizers, for example
+// LSan will not work with __auxv_init_environ(),
+// since it needs getauxval() before.
+//
 static size_t __find_auxv(unsigned long type)
 {
     size_t i;
-    for (i = 0; __auxv[i]; i += 2)
+    for (i = 0; __auxv_environ[i]; i += 2)
     {
-        if (__auxv[i] == type)
+        if (__auxv_environ[i] == type)
+        {
             return i + 1;
+        }
     }
     return (size_t) -1;
 }
-
-unsigned long __getauxval(unsigned long type)
+unsigned long __getauxval_environ(unsigned long type)
 {
     if (type == AT_SECURE)
         return __auxv_secure;
 
-    if (__auxv)
+    if (__auxv_environ)
     {
         size_t index = __find_auxv(type);
         if (index != ((size_t) -1))
-            return __auxv[index];
+            return __auxv_environ[index];
     }
 
     errno = ENOENT;
     return 0;
 }
-
-static void * volatile getauxval_func;
-
-static unsigned long  __auxv_init(unsigned long type)
+static unsigned long  __auxv_init_environ(unsigned long type)
 {
     if (!__environ)
     {
-        // __environ is not initialized yet so we can't initialize __auxv right now.
+        // __environ is not initialized yet so we can't initialize __auxv_environ right now.
         // That's normally occurred only when getauxval() is called from some sanitizer's internal code.
         errno = ENOENT;
         return 0;
     }
 
-    // Initialize __auxv and __auxv_secure.
+    // Initialize __auxv_environ and __auxv_secure.
     size_t i;
     for (i = 0; __environ[i]; i++);
-    __auxv = (unsigned long *) (__environ + i + 1);
+    __auxv_environ = (unsigned long *) (__environ + i + 1);
 
     size_t secure_idx = __find_auxv(AT_SECURE);
     if (secure_idx != ((size_t) -1))
-        __auxv_secure = __auxv[secure_idx];
+        __auxv_secure = __auxv_environ[secure_idx];
 
-    // Now we've initialized __auxv, next time getauxval() will only call __get_auxval().
-    a_cas_p(&getauxval_func, (void *)__auxv_init, (void *)__getauxval);
+    // Now we need to switch to __getauxval_environ for all later calls, since
+    // everything is initialized.
+    a_cas_p(&getauxval_func, (void *)__auxv_init_environ, (void *)__getauxval_environ);
 
-    return __getauxval(type);
+    return __getauxval_environ(type);
 }
 
-// First time getauxval() will call __auxv_init().
-static void * volatile getauxval_func = (void *)__auxv_init;
+// Callchain:
+// - __auxv_init_procfs -> __getauxval_environ
+// - __auxv_init_procfs -> __auxv_init_environ -> __getauxval_environ
+static void * volatile getauxval_func = (void *)__auxv_init_procfs;
 
 unsigned long getauxval(unsigned long type)
 {

cmake/cpu_features.cmake

@@ -19,6 +19,7 @@ option (ENABLE_POPCNT "Use popcnt instructions on x86_64" 1)
 option (ENABLE_AVX "Use AVX instructions on x86_64" 0)
 option (ENABLE_AVX2 "Use AVX2 instructions on x86_64" 0)
 option (ENABLE_AVX512 "Use AVX512 instructions on x86_64" 0)
+option (ENABLE_AVX512_VBMI "Use AVX512_VBMI instruction on x86_64 (depends on ENABLE_AVX512)" 0)
 option (ENABLE_BMI "Use BMI instructions on x86_64" 0)
 option (ENABLE_AVX2_FOR_SPEC_OP "Use avx2 instructions for specific operations on x86_64" 0)
 option (ENABLE_AVX512_FOR_SPEC_OP "Use avx512 instructions for specific operations on x86_64" 0)
@@ -151,6 +152,20 @@ elseif (ARCH_AMD64)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
     endif ()
 
+    set (TEST_FLAG "-mavx512vbmi")
+    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
+    check_cxx_source_compiles("
+        #include <immintrin.h>
+        int main() {
+            auto a = _mm512_permutexvar_epi8(__m512i(), __m512i());
+            (void)a;
+            return 0;
+        }
+    " HAVE_AVX512_VBMI)
+    if (HAVE_AVX512 AND ENABLE_AVX512 AND HAVE_AVX512_VBMI AND ENABLE_AVX512_VBMI)
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
+    endif ()
+
     set (TEST_FLAG "-mbmi")
     set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
     check_cxx_source_compiles("

cmake/warnings.cmake

@@ -20,16 +20,11 @@ if (COMPILER_CLANG)
     # We want to get everything out of the compiler for code quality.
     add_warning(everything)
     add_warning(pedantic)
-    no_warning(vla-extension)
     no_warning(zero-length-array)
-    no_warning(c11-extensions)
-    no_warning(unused-command-line-argument)
     no_warning(c++98-compat-pedantic)
     no_warning(c++98-compat)
-    no_warning(c99-extensions)
     no_warning(conversion)
     no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
-    no_warning(deprecated-dynamic-exception-spec)
     no_warning(disabled-macro-expansion)
     no_warning(documentation-unknown-command)
     no_warning(double-promotion)
@@ -38,12 +33,7 @@ if (COMPILER_CLANG)
     no_warning(global-constructors)
     no_warning(missing-prototypes)
     no_warning(missing-variable-declarations)
-    no_warning(nested-anon-types)
-    no_warning(packed)
     no_warning(padded)
-    no_warning(return-std-move-in-c++11) # clang 7+
-    no_warning(shift-sign-overflow)
-    no_warning(sign-conversion)
     no_warning(switch-enum)
     no_warning(undefined-func-template)
     no_warning(unused-template)

contrib/googletest-cmake/CMakeLists.txt

@@ -2,7 +2,7 @@ set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest")
 
 add_library(_gtest "${SRC_DIR}/src/gtest-all.cc")
 set_target_properties(_gtest PROPERTIES VERSION "1.0.0")
-target_compile_definitions (_gtest INTERFACE GTEST_HAS_POSIX_RE=0)
+target_compile_definitions (_gtest PUBLIC GTEST_HAS_POSIX_RE=0)
 target_include_directories(_gtest SYSTEM PUBLIC "${SRC_DIR}/include")
 target_include_directories(_gtest PRIVATE "${SRC_DIR}")
 

contrib/thrift

@@ -1 +1 @@
-Subproject commit 010ccf0a0c7023fea0f6bf4e4078ebdff7e61982
+Subproject commit 2a93df80f27739ccabb5b885cb12a8dc7595ecdf

contrib/thrift-cmake/CMakeLists.txt

@@ -15,7 +15,6 @@ set(thriftcpp_SOURCES
         "${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.cpp"
         "${LIBRARY_DIR}/src/thrift/concurrency/ThreadManager.cpp"
         "${LIBRARY_DIR}/src/thrift/concurrency/TimerManager.cpp"
-        "${LIBRARY_DIR}/src/thrift/concurrency/Util.cpp"
         "${LIBRARY_DIR}/src/thrift/processor/PeekProcessor.cpp"
         "${LIBRARY_DIR}/src/thrift/protocol/TBase64Utils.cpp"
         "${LIBRARY_DIR}/src/thrift/protocol/TDebugProtocol.cpp"
@@ -33,6 +32,8 @@ set(thriftcpp_SOURCES
         "${LIBRARY_DIR}/src/thrift/transport/TServerSocket.cpp"
         "${LIBRARY_DIR}/src/thrift/transport/TTransportUtils.cpp"
         "${LIBRARY_DIR}/src/thrift/transport/TBufferTransports.cpp"
+        "${LIBRARY_DIR}/src/thrift/transport/SocketCommon.cpp"
+        "${LIBRARY_DIR}/src/thrift/transport/TWebSocketServer.cpp"
         "${LIBRARY_DIR}/src/thrift/server/TConnectedClient.cpp"
         "${LIBRARY_DIR}/src/thrift/server/TServerFramework.cpp"
         "${LIBRARY_DIR}/src/thrift/server/TSimpleServer.cpp"
@@ -92,4 +93,4 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 add_library(_thrift ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
 add_library(ch_contrib::thrift ALIAS _thrift)
 target_include_directories(_thrift SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src" ${CMAKE_CURRENT_BINARY_DIR})
-target_link_libraries (_thrift PUBLIC boost::headers_only)
+target_link_libraries (_thrift PUBLIC OpenSSL::SSL boost::headers_only)

docker/packager/binary/build.sh

@@ -29,17 +29,21 @@ env
 
 if [ -n "$MAKE_DEB" ]; then
   rm -rf /build/packages/root
-  if [ -z "$SANITIZER" ]; then
-    # We need to check if clickhouse-diagnostics is fine and build it
-    (
-      cd /build/programs/diagnostics
-      make test-no-docker
-      GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build
-      mv clickhouse-diagnostics ..
-    )
-  else
-    echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics
-    chmod +x /build/programs/clickhouse-diagnostics
+  # NOTE: this is for backward compatibility with previous releases,
+  # that does not diagnostics tool (only script).
+  if [ -d /build/programs/diagnostics ]; then
+    if [ -z "$SANITIZER" ]; then
+      # We need to check if clickhouse-diagnostics is fine and build it
+      (
+        cd /build/programs/diagnostics
+        make test-no-docker
+        GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build
+        mv clickhouse-diagnostics ..
+      )
+    else
+      echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics
+      chmod +x /build/programs/clickhouse-diagnostics
+    fi
   fi
 fi
 

docker/test/fasttest/Dockerfile

@@ -71,6 +71,7 @@ RUN apt-get update \
         software-properties-common \
         tzdata \
         unixodbc \
+        file \
        --yes --no-install-recommends
 
 RUN pip3 install numpy scipy pandas Jinja2

docker/test/stateless/Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update -y \
             unixodbc \
             wget \
             zstd \
+            file \
     && apt-get clean
 
 

docker/test/stress/run.sh

@@ -322,7 +322,11 @@ else
     clickhouse-client --query="SELECT 'Server version: ', version()"
 
     # Install new package before running stress test because we should use new clickhouse-client and new clickhouse-test
+    # But we should leave old binary in /usr/bin/ for gdb (so it will print sane stacktarces)
+    mv /usr/bin/clickhouse previous_release_package_folder/
     install_packages package_folder
+    mv /usr/bin/clickhouse package_folder/
+    mv previous_release_package_folder/clickhouse /usr/bin/
 
     mkdir tmp_stress_output
 
@@ -337,6 +341,7 @@ else
     mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
 
     # Start new server
+    mv package_folder/clickhouse /usr/bin/
     configure
     start 500
     clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
@@ -354,6 +359,11 @@ else
     # Error messages (we should ignore some errors)
     # FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.")
     # FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64")
+    # FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
+    #       Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
+    # FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
+    # NOTE  Incompatibility was introduced in https://github.com/ClickHouse/ClickHouse/pull/39263, it's expected
+    #       ("This engine is deprecated and is not supported in transactions", "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part")
     echo "Check for Error messages in server log:"
     zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
                -e "Code: 236. DB::Exception: Cancelled mutating parts" \
@@ -378,6 +388,11 @@ else
                -e "is lost forever." \
                -e "Unknown index: idx." \
                -e "Cannot parse string 'Hello' as UInt64" \
+               -e "} <Error> TCPHandler: Code:" \
+               -e "} <Error> executeQuery: Code:" \
+               -e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
+               -e "This engine is deprecated and is not supported in transactions" \
+               -e "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part" \
         /var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
         && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
         || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv

docs/_includes/install/universal.sh

@@ -7,31 +7,31 @@ DIR=
 
 if [ "${OS}" = "Linux" ]
 then
-    if [ "${ARCH}" = "x86_64" ]
+    if [ "${ARCH}" = "x86_64" -o "${ARCH}" = "amd64" ]
     then
         DIR="amd64"
-    elif [ "${ARCH}" = "aarch64" ]
+    elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ]
     then
         DIR="aarch64"
-    elif [ "${ARCH}" = "powerpc64le" ] || [ "${ARCH}" = "ppc64le" ]
+    elif [ "${ARCH}" = "powerpc64le" -o "${ARCH}" = "ppc64le" ]
     then
         DIR="powerpc64le"
     fi
 elif [ "${OS}" = "FreeBSD" ]
 then
-    if [ "${ARCH}" = "x86_64" ]
+    if [ "${ARCH}" = "x86_64" -o "${ARCH}" = "amd64" ]
     then
         DIR="freebsd"
-    elif [ "${ARCH}" = "aarch64" ]
+    elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ]
     then
         DIR="freebsd-aarch64"
-    elif [ "${ARCH}" = "powerpc64le" ] || [ "${ARCH}" = "ppc64le" ]
+    elif [ "${ARCH}" = "powerpc64le" -o "${ARCH}" = "ppc64le" ]
     then
         DIR="freebsd-powerpc64le"
     fi
 elif [ "${OS}" = "Darwin" ]
 then
-    if [ "${ARCH}" = "x86_64" ]
+    if [ "${ARCH}" = "x86_64" -o "${ARCH}" = "amd64" ]
     then
         DIR="macos"
     elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ]

docs/en/interfaces/formats.md

@@ -1632,6 +1632,8 @@ kafka_topic_list = 'topic1',
 kafka_group_name = 'group1',
 kafka_format = 'AvroConfluent';
 
+-- for debug purposes you can set format_avro_schema_registry_url in a session.
+-- this way cannot be used in production
 SET format_avro_schema_registry_url = 'http://schema-registry';
 
 SELECT * FROM topic1_stream;

docs/en/operations/settings/settings.md

@@ -2626,7 +2626,7 @@ Possible values:
 -   Any positive integer.
 -   0 - Disabled (infinite timeout).
 
-Default value: 1800.
+Default value: 180.
 
 ## http_receive_timeout {#http_receive_timeout}
 
@@ -2637,7 +2637,7 @@ Possible values:
 -   Any positive integer.
 -   0 - Disabled (infinite timeout).
 
-Default value: 1800.
+Default value: 180.
 
 ## check_query_single_value_result {#check_query_single_value_result}
 

docs/en/operations/system-tables/part_log.md

@@ -19,6 +19,10 @@ The `system.part_log` table contains the following columns:
     -   `REGULAR_MERGE` — Some regular merge.
     -   `TTL_DELETE_MERGE` — Cleaning up expired data.
     -   `TTL_RECOMPRESS_MERGE` — Recompressing data part with the.
+-   `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Merge algorithm for the event with type `MERGE_PARTS`. Can have one of the following values:
+    -   `UNDECIDED`
+    -   `HORIZONTAL`
+    -   `VERTICAL`
 -   `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
 -   `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
 -   `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision.
@@ -52,6 +56,7 @@ Row 1:
 query_id:                      983ad9c7-28d5-4ae1-844e-603116b7de31
 event_type:                    NewPart
 merge_reason:                  NotAMerge
+merge_algorithm:               Undecided
 event_date:                    2021-02-02
 event_time:                    2021-02-02 11:14:28
 event_time_microseconds:                    2021-02-02 11:14:28.861919

docs/en/sql-reference/functions/array-functions.md

@@ -1715,3 +1715,7 @@ Return value type is always [Float64](../../sql-reference/data-types/float.md).
 │ 6   │ Float64                                                                                  │
 └─────┴──────────────────────────────────────────────────────────────────────────────────────────┘
 ```
+
+## Distance functions
+
+All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).

docs/en/sql-reference/functions/distance-functions.md

@@ -0,0 +1,502 @@
+# Distance functions
+
+## L1Norm
+
+Calculates the sum of absolute values of a vector.
+
+**Syntax**
+
+```sql
+L1Norm(vector)
+```
+
+Alias: `normL1`.
+
+**Arguments**
+
+-   `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance.
+
+Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
+
+**Examples**
+
+Query:
+
+```sql
+SELECT L1Norm((1, 2));
+```
+
+Result:
+
+```text
+┌─L1Norm((1, 2))─┐
+│              3 │
+└────────────────┘
+```
+
+## L2Norm
+
+Calculates the square root of the sum of the squares of the vector values.
+
+**Syntax**
+
+```sql
+L2Norm(vector)
+```
+
+Alias: `normL2`.
+
+**Arguments**
+
+-   `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance).
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L2Norm((1, 2));
+```
+
+Result:
+
+```text
+┌───L2Norm((1, 2))─┐
+│ 2.23606797749979 │
+└──────────────────┘
+```
+
+## LinfNorm
+
+Calculates the maximum of absolute values of a vector.
+
+**Syntax**
+
+```sql
+LinfNorm(vector)
+```
+
+Alias: `normLinf`.
+
+**Arguments**
+
+-   `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   Linf-norm or the maximum absolute value.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LinfNorm((1, -2));
+```
+
+Result:
+
+```text
+┌─LinfNorm((1, -2))─┐
+│                 2 │
+└───────────────────┘
+```
+
+## LpNorm
+
+Calculates the root of `p`-th power of the sum of the absolute values of a vector in the power of `p`.
+
+**Syntax**
+
+```sql
+LpNorm(vector, p)
+```
+
+Alias: `normLp`.
+
+**Arguments**
+
+-   `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `p` — The power. Possible values: real number in `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
+
+**Returned value**
+
+-   [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LpNorm((1, -2), 2);
+```
+
+Result:
+
+```text
+┌─LpNorm((1, -2), 2)─┐
+│   2.23606797749979 │
+└────────────────────┘
+```
+
+## L1Distance
+
+Calculates the distance between two points (the values of the vectors are the coordinates) in `L1` space (1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance)).
+
+**Syntax**
+
+```sql
+L1Distance(vector1, vector2)
+```
+
+Alias: `distanceL1`.
+
+**Arguments**
+
+-   `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   1-norm distance.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L1Distance((1, 2), (2, 3));
+```
+
+Result:
+
+```text
+┌─L1Distance((1, 2), (2, 3))─┐
+│                          2 │
+└────────────────────────────┘
+```
+
+## L2Distance
+
+Calculates the distance between two points (the values of the vectors are the coordinates) in Euclidean space ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
+
+**Syntax**
+
+```sql
+L2Distance(vector1, vector2)
+```
+
+Alias: `distanceL2`.
+
+**Arguments**
+
+-   `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   2-norm distance.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L2Distance((1, 2), (2, 3));
+```
+
+Result:
+
+```text
+┌─L2Distance((1, 2), (2, 3))─┐
+│         1.4142135623730951 │
+└────────────────────────────┘
+```
+
+## LinfDistance
+
+Calculates the distance between two points (the values of the vectors are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
+
+**Syntax**
+
+```sql
+LinfDistance(vector1, vector2)
+```
+
+Alias: `distanceLinf`.
+
+**Arguments**
+
+-   `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `vector1` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   Infinity-norm distance.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LinfDistance((1, 2), (2, 3));
+```
+
+Result:
+
+```text
+┌─LinfDistance((1, 2), (2, 3))─┐
+│                            1 │
+└──────────────────────────────┘
+```
+
+## LpDistance
+
+Calculates the distance between two points (the values of the vectors are the coordinates) in `Lp` space ([p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
+
+**Syntax**
+
+```sql
+LpDistance(vector1, vector2, p)
+```
+
+Alias: `distanceLp`.
+
+**Arguments**
+
+-   `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `p` — The power. Possible values: real number from `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
+
+**Returned value**
+
+-   p-norm distance.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LpDistance((1, 2), (2, 3), 3);
+```
+
+Result:
+
+```text
+┌─LpDistance((1, 2), (2, 3), 3)─┐
+│            1.2599210498948732 │
+└───────────────────────────────┘
+```
+
+
+## L1Normalize
+
+Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L1` space ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)).
+
+**Syntax**
+
+```sql
+L1Normalize(tuple)
+```
+
+Alias: `normalizeL1`.
+
+**Arguments**
+
+-   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
+
+**Returned value**
+
+-   Unit vector.
+
+Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L1Normalize((1, 2));
+```
+
+Result:
+
+```text
+┌─L1Normalize((1, 2))─────────────────────┐
+│ (0.3333333333333333,0.6666666666666666) │
+└─────────────────────────────────────────┘
+```
+
+## L2Normalize
+
+Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in Euclidean space (using [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
+
+**Syntax**
+
+```sql
+L2Normalize(tuple)
+```
+
+Alias: `normalizeL1`.
+
+**Arguments**
+
+-   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
+
+**Returned value**
+
+-   Unit vector.
+
+Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L2Normalize((3, 4));
+```
+
+Result:
+
+```text
+┌─L2Normalize((3, 4))─┐
+│ (0.6,0.8)           │
+└─────────────────────┘
+```
+
+## LinfNormalize
+
+Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L_{inf}` space (using [maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
+
+**Syntax**
+
+```sql
+LinfNormalize(tuple)
+```
+
+Alias: `normalizeLinf `.
+
+**Arguments**
+
+-   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
+
+**Returned value**
+
+-   Unit vector.
+
+Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LinfNormalize((3, 4));
+```
+
+Result:
+
+```text
+┌─LinfNormalize((3, 4))─┐
+│ (0.75,1)              │
+└───────────────────────┘
+```
+
+## LpNormalize
+
+Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `Lp` space (using [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
+
+**Syntax**
+
+```sql
+LpNormalize(tuple, p)
+```
+
+Alias: `normalizeLp `.
+
+**Arguments**
+
+-   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
+-   `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
+
+**Returned value**
+
+-   Unit vector.
+
+Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LpNormalize((3, 4),5);
+```
+
+Result:
+
+```text
+┌─LpNormalize((3, 4), 5)──────────────────┐
+│ (0.7187302630182624,0.9583070173576831) │
+└─────────────────────────────────────────┘
+```
+
+## cosineDistance
+
+Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors.
+
+**Syntax**
+
+```sql
+cosineDistance(tuple1, tuple2)
+```
+
+**Arguments**
+
+-   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
+-   `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
+
+**Returned value**
+
+-   Cosine of the angle between two vectors substracted from one.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT cosineDistance((1, 2), (2, 3));
+```
+
+Result:
+
+```text
+┌─cosineDistance((1, 2), (2, 3))─┐
+│           0.007722123286332261 │
+└────────────────────────────────┘
+```

docs/en/sql-reference/functions/other-functions.md

@@ -681,6 +681,47 @@ SELECT
 └────────────┴─────────────────────────────────────────────────────────────────┘
 ```
 
+## parseTimeDelta
+
+Parse a sequence of numbers followed by something resembling a time unit.
+
+**Syntax**
+
+```sql
+parseTimeDelta(timestr)
+```
+
+**Arguments**
+
+-   `timestr` — A sequence of numbers followed by something resembling a time unit.
+
+
+**Returned value**
+
+-   A floating-point number with the number of seconds.
+
+**Example**
+
+```sql
+SELECT parseTimeDelta('11s+22min')
+```
+
+```text
+┌─parseTimeDelta('11s+22min')─┐
+│                        1331 │
+└─────────────────────────────┘
+```
+
+```sql
+SELECT parseTimeDelta('1yr2mo')
+```
+
+```text
+┌─parseTimeDelta('1yr2mo')─┐
+│                 36806400 │
+└──────────────────────────┘
+```
+
 ## least(a, b)
 
 Returns the smallest value from a and b.

docs/en/sql-reference/functions/string-replace-functions.md

@@ -93,3 +93,35 @@ Predefined characters: `\0`, `\\`, `|`, `(`, `)`, `^`, `$`, `.`, `[`, `]`, `?`,
 This implementation slightly differs from re2::RE2::QuoteMeta. It escapes zero byte as `\0` instead of `\x00` and it escapes only required characters.
 For more information, see the link: [RE2](https://github.com/google/re2/blob/master/re2/re2.cc#L473)
 
+
+## translate(s, from, to)
+
+The function replaces characters in the string ‘s’ in accordance with one-to-one character mapping defined by ‘from’ and ‘to’ strings. ‘from’ and ‘to’ must be ASCII strings of the same size. Non-ASCII characters in the original string are not modified.
+
+Example:
+
+``` sql
+SELECT translate('Hello, World!', 'delor', 'DELOR') AS res
+```
+
+``` text
+┌─res───────────┐
+│ HELLO, WORLD! │
+└───────────────┘
+```
+
+## translateUTF8(string, from, to)
+
+Similar to previous function, but works with UTF-8 arguments. ‘from’ and ‘to’ must be valid UTF-8 strings of the same size.
+
+Example:
+
+``` sql
+SELECT translateUTF8('Hélló, Wórld¡', 'óé¡', 'oe!') AS res
+```
+
+``` text
+┌─res───────────┐
+│ Hello, World! │
+└───────────────┘
+```

docs/en/sql-reference/functions/string-search-functions.md

@@ -374,7 +374,7 @@ The same as `multiMatchAny`, but returns any index that matches the haystack.
 
 ## multiMatchAllIndices(haystack, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
 
-The same as `multiMatchAny`, but returns the array of all indicies that match the haystack in any order.
+The same as `multiMatchAny`, but returns the array of all indices that match the haystack in any order.
 
 ## multiFuzzyMatchAny(haystack, distance, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
 

docs/en/sql-reference/functions/tuple-functions.md

@@ -559,502 +559,7 @@ Result:
 └────────────────────────────┘
 ```
 
-## L1Norm
 
-Calculates the sum of absolute values of a tuple.
+## Distance functions
 
-**Syntax**
-
-```sql
-L1Norm(tuple)
-```
-
-Alias: `normL1`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance.
-
-Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L1Norm((1, 2));
-```
-
-Result:
-
-```text
-┌─L1Norm((1, 2))─┐
-│              3 │
-└────────────────┘
-```
-
-## L2Norm
-
-Calculates the square root of the sum of the squares of the tuple values.
-
-**Syntax**
-
-```sql
-L2Norm(tuple)
-```
-
-Alias: `normL2`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance).
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L2Norm((1, 2));
-```
-
-Result:
-
-```text
-┌───L2Norm((1, 2))─┐
-│ 2.23606797749979 │
-└──────────────────┘
-```
-
-## LinfNorm
-
-Calculates the maximum of absolute values of a tuple.
-
-**Syntax**
-
-```sql
-LinfNorm(tuple)
-```
-
-Alias: `normLinf`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Linf-norm or the maximum absolute value.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LinfNorm((1, -2));
-```
-
-Result:
-
-```text
-┌─LinfNorm((1, -2))─┐
-│                 2 │
-└───────────────────┘
-```
-
-## LpNorm
-
-Calculates the root of `p`-th power of the sum of the absolute values of a tuple in the power of `p`.
-
-**Syntax**
-
-```sql
-LpNorm(tuple, p)
-```
-
-Alias: `normLp`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
--   `p` — The power. Possible values: real number in `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
-
-**Returned value**
-
--   [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LpNorm((1, -2), 2);
-```
-
-Result:
-
-```text
-┌─LpNorm((1, -2), 2)─┐
-│   2.23606797749979 │
-└────────────────────┘
-```
-
-## L1Distance
-
-Calculates the distance between two points (the values of the tuples are the coordinates) in `L1` space (1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance)).
-
-**Syntax**
-
-```sql
-L1Distance(tuple1, tuple2)
-```
-
-Alias: `distanceL1`.
-
-**Arguments**
-
--   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   1-norm distance.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L1Distance((1, 2), (2, 3));
-```
-
-Result:
-
-```text
-┌─L1Distance((1, 2), (2, 3))─┐
-│                          2 │
-└────────────────────────────┘
-```
-
-## L2Distance
-
-Calculates the distance between two points (the values of the tuples are the coordinates) in Euclidean space ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
-
-**Syntax**
-
-```sql
-L2Distance(tuple1, tuple2)
-```
-
-Alias: `distanceL2`.
-
-**Arguments**
-
--   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   2-norm distance.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L2Distance((1, 2), (2, 3));
-```
-
-Result:
-
-```text
-┌─L2Distance((1, 2), (2, 3))─┐
-│         1.4142135623730951 │
-└────────────────────────────┘
-```
-
-## LinfDistance
-
-Calculates the distance between two points (the values of the tuples are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
-
-**Syntax**
-
-```sql
-LinfDistance(tuple1, tuple2)
-```
-
-Alias: `distanceLinf`.
-
-**Arguments**
-
--   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Infinity-norm distance.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LinfDistance((1, 2), (2, 3));
-```
-
-Result:
-
-```text
-┌─LinfDistance((1, 2), (2, 3))─┐
-│                            1 │
-└──────────────────────────────┘
-```
-
-## LpDistance
-
-Calculates the distance between two points (the values of the tuples are the coordinates) in `Lp` space ([p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
-
-**Syntax**
-
-```sql
-LpDistance(tuple1, tuple2, p)
-```
-
-Alias: `distanceLp`.
-
-**Arguments**
-
--   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `p` — The power. Possible values: real number from `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
-
-**Returned value**
-
--   p-norm distance.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LpDistance((1, 2), (2, 3), 3);
-```
-
-Result:
-
-```text
-┌─LpDistance((1, 2), (2, 3), 3)─┐
-│            1.2599210498948732 │
-└───────────────────────────────┘
-```
-
-## L1Normalize
-
-Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L1` space ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)).
-
-**Syntax**
-
-```sql
-L1Normalize(tuple)
-```
-
-Alias: `normalizeL1`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Unit vector.
-
-Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L1Normalize((1, 2));
-```
-
-Result:
-
-```text
-┌─L1Normalize((1, 2))─────────────────────┐
-│ (0.3333333333333333,0.6666666666666666) │
-└─────────────────────────────────────────┘
-```
-
-## L2Normalize
-
-Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in Euclidean space (using [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
-
-**Syntax**
-
-```sql
-L2Normalize(tuple)
-```
-
-Alias: `normalizeL1`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Unit vector.
-
-Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L2Normalize((3, 4));
-```
-
-Result:
-
-```text
-┌─L2Normalize((3, 4))─┐
-│ (0.6,0.8)           │
-└─────────────────────┘
-```
-
-## LinfNormalize
-
-Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L_{inf}` space (using [maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
-
-**Syntax**
-
-```sql
-LinfNormalize(tuple)
-```
-
-Alias: `normalizeLinf `.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Unit vector.
-
-Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LinfNormalize((3, 4));
-```
-
-Result:
-
-```text
-┌─LinfNormalize((3, 4))─┐
-│ (0.75,1)              │
-└───────────────────────┘
-```
-
-## LpNormalize
-
-Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `Lp` space (using [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
-
-**Syntax**
-
-```sql
-LpNormalize(tuple, p)
-```
-
-Alias: `normalizeLp `.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
--   `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
-
-**Returned value**
-
--   Unit vector.
-
-Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LpNormalize((3, 4),5);
-```
-
-Result:
-
-```text
-┌─LpNormalize((3, 4), 5)──────────────────┐
-│ (0.7187302630182624,0.9583070173576831) │
-└─────────────────────────────────────────┘
-```
-
-## cosineDistance
-
-Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors.
-
-**Syntax**
-
-```sql
-cosineDistance(tuple1, tuple2)
-```
-
-**Arguments**
-
--   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Cosine of the angle between two vectors substracted from one.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT cosineDistance((1, 2), (2, 3));
-```
-
-Result:
-
-```text
-┌─cosineDistance((1, 2), (2, 3))─┐
-│           0.007722123286332261 │
-└────────────────────────────────┘
-```
+All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).

docs/en/sql-reference/statements/alter/partition.md

@@ -108,7 +108,8 @@ Note that data will be deleted neither from `table1` nor from `table2`.
 For the query to run successfully, the following conditions must be met:
 
 -   Both tables must have the same structure.
--   Both tables must have the same partition key.
+-   Both tables must have the same partition key, the same order by key and the same primary key.
+-   Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables).
 
 ## REPLACE PARTITION
 
@@ -121,7 +122,8 @@ This query copies the data partition from the `table1` to `table2` and replaces
 For the query to run successfully, the following conditions must be met:
 
 -   Both tables must have the same structure.
--   Both tables must have the same partition key.
+-   Both tables must have the same partition key, the same order by key and the same primary key.
+-   Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables).
 
 ## MOVE PARTITION TO TABLE
 
@@ -134,9 +136,9 @@ This query moves the data partition from the `table_source` to `table_dest` with
 For the query to run successfully, the following conditions must be met:
 
 -   Both tables must have the same structure.
--   Both tables must have the same partition key.
+-   Both tables must have the same partition key, the same order by key and the same primary key.
+-   Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables).
 -   Both tables must be the same engine family (replicated or non-replicated).
--   Both tables must have the same storage policy.
 
 ## CLEAR COLUMN IN PARTITION
 

docs/en/sql-reference/table-functions/null.md

@@ -5,7 +5,7 @@ sidebar_label: null function
 
 # null
 
-Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately droped right after the query execution. The function is used for the convenience of test writing and demonstrations.
+Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately dropped right after the query execution. The function is used for the convenience of test writing and demonstrations.
 
 **Syntax**
 
@@ -40,4 +40,3 @@ See also:
 
 -   [Null table engine](../../engines/table-engines/special/null.md)
 
-[Original article](https://clickhouse.com/docs/en/sql-reference/table-functions/null/) <!--hide-->

docs/en/sql-reference/table-functions/remote.md

@@ -5,11 +5,11 @@ sidebar_label: remote
 
 # remote, remoteSecure
 
-Allows to access remote servers without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. `remoteSecure` - same as `remote` but with a secured connection.
+Allows accessing remote servers, including migration of data, without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. `remoteSecure` - same as `remote` but with a secured connection.
 
 Both functions can be used in `SELECT` and `INSERT` queries.
 
-**Syntax**
+## Syntax
 
 ``` sql
 remote('addresses_expr', db, table[, 'user'[, 'password'], sharding_key])
@@ -18,7 +18,7 @@ remoteSecure('addresses_expr', db, table[, 'user'[, 'password'], sharding_key])
 remoteSecure('addresses_expr', db.table[, 'user'[, 'password'], sharding_key])
 ```
 
-**Parameters**
+## Parameters
 
 - `addresses_expr` — An expression that generates addresses of remote servers. This may be just one server address. The server address is `host:port`, or just `host`.
 
@@ -36,28 +36,31 @@ remoteSecure('addresses_expr', db.table[, 'user'[, 'password'], sharding_key])
 - `password` — User password. If the password is not specified, an empty password is used. Type: [String](../../sql-reference/data-types/string.md).
 - `sharding_key` — Sharding key to support distributing data across nodes. For example: `insert into remote('127.0.0.1:9000,127.0.0.2', db, table, 'default', rand())`. Type: [UInt32](../../sql-reference/data-types/int-uint.md).
 
-**Returned value**
+## Returned value
 
 The dataset from remote servers.
 
-**Usage**
+## Usage
 
-Using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function.
+Unless you are migrating data from one system to another, using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function.
 
 The `remote` table function can be useful in the following cases:
 
--   Accessing a specific server for data comparison, debugging, and testing.
--   Queries between various ClickHouse clusters for research purposes.
--   Infrequent distributed requests that are made manually.
--   Distributed requests where the set of servers is re-defined each time.
+- Migrating data from one system to another
+- Accessing a specific server for data comparison, debugging, and testing.
+- Queries between various ClickHouse clusters for research purposes.
+- Infrequent distributed requests that are made manually.
+- Distributed requests where the set of servers is re-defined each time.
 
-**Adresses**
+### Adresses
 
 ``` text
 example01-01-1
+example01-01-1:9440
 example01-01-1:9000
 localhost
 127.0.0.1
+[::]:9440
 [::]:9000
 [2a02:6b8:0:1111::11]:9000
 ```
@@ -68,15 +71,15 @@ Multiple addresses can be comma-separated. In this case, ClickHouse will use dis
 example01-01-1,example01-02-1
 ```
 
-**Examples**
+## Examples
 
-Selecting data from a remote server:
+### Selecting data from a remote server:
 
 ``` sql
 SELECT * FROM remote('127.0.0.1', db.remote_engine_table) LIMIT 3;
 ```
 
-Inserting data from a remote server into a table:
+### Inserting data from a remote server into a table:
 
 ``` sql
 CREATE TABLE remote_table (name String, value UInt32) ENGINE=Memory;
@@ -84,7 +87,65 @@ INSERT INTO FUNCTION remote('127.0.0.1', currentDatabase(), 'remote_table') VALU
 SELECT * FROM remote_table;
 ```
 
-## Globs in Addresses {globs-in-addresses}
+### Migration of tables from one system to another:
+This example uses one table from a sample dataset.  The database is `imdb`, and the table is `actors`. 
+
+#### On the source ClickHouse system (the system that currently hosts the data)
+- Verify the source database and table name (`imdb.actors`)  
+  ```sql
+  show databases
+  ```
+
+  ```sql
+  show tables in imdb
+  ```
+
+- Get the CREATE TABLE statement from the source:
+  ```
+  select create_table_query
+  from system.tables
+  where database = 'imdb' and table = 'actors'
+  ```
+
+  Response
+  ```sql
+  CREATE TABLE imdb.actors (`id` UInt32,
+                            `first_name` String,
+                            `last_name` String,
+                            `gender` FixedString(1))
+                  ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
+                  ORDER BY (id, first_name, last_name, gender)
+                  SETTINGS index_granularity = 8192
+  ```
+
+#### On the destination ClickHouse system:
+
+- Create the destination database:
+  ```sql
+  CREATE DATABASE imdb
+  ```
+
+- Using the CREATE TABLE statement from the source, create the destination:
+  ```sql
+  CREATE TABLE imdb.actors (`id` UInt32,
+                            `first_name` String,
+                            `last_name` String,
+                            `gender` FixedString(1))
+                  ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
+                  ORDER BY (id, first_name, last_name, gender)
+                  SETTINGS index_granularity = 8192
+  ```
+
+#### Back on the source deployment:
+
+Insert into the new database and table created on the remote system.  You will need the host, port, username, password, destination database, and destination table.
+```sql
+INSERT INTO FUNCTION
+remoteSecure('remote.clickhouse.cloud:9440', 'imdb.actors', 'USER', 'PASSWORD', rand())
+SELECT * from imdb.actors
+```
+
+## Globs in Addresses {#globs-in-addresses}
 
 Patterns in curly brackets `{ }` are used to generate a set of shards and to specify replicas. If there are multiple pairs of curly brackets, then the direct product of the corresponding sets is generated.
 The following pattern types are supported.

docs/ru/engines/table-engines/mergetree-family/mergetree.md

@@ -335,7 +335,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234
 
     Поддерживаемые типы данных: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`.
 
-    Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem).
+    Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions#hasany), [hasAll](../../../sql-reference/functions/array-functions#hasall).
 
 **Примеры**
 

docs/ru/operations/system-tables/part_log.md

@@ -14,6 +14,15 @@
     -   `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
     -   `MUTATE_PART` — изменение куска.
     -   `MOVE_PART` — перемещение куска между дисками.
+-   `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — Причина события с типом `MERGE_PARTS`. Может принимать одно из следующих значений:
+    -   `NOT_A_MERGE` — событие имеет тип иной, чем `MERGE_PARTS`.
+    -   `REGULAR_MERGE` — обычное слияние.
+    -   `TTL_DELETE_MERGE` — очистка истекших данных.
+    -   `TTL_RECOMPRESS_MERGE` — переупаковка куска.
+-   `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Алгоритм слияния для события с типом `MERGE_PARTS`. Может принимать одно из следующих значений:
+    -   `UNDECIDED`
+    -   `HORIZONTAL`
+    -   `VERTICAL`
 -   `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата события.
 -   `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время события.
 -   `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время события с точностью до микросекунд.
@@ -46,6 +55,8 @@ Row 1:
 ──────
 query_id:                      983ad9c7-28d5-4ae1-844e-603116b7de31
 event_type:                    NewPart
+merge_reason:                  NotAMerge
+merge_algorithm:               Undecided
 event_date:                    2021-02-02
 event_time:                    2021-02-02 11:14:28
 event_time_microseconds:                    2021-02-02 11:14:28.861919

docs/ru/sql-reference/functions/string-replace-functions.md

@@ -83,3 +83,34 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res
 └─────────────────────┘
 ```
 
+## translate(s, from, to)
+
+Данная функция заменяет символы в строке ‘s’ в соответствии с поэлементным отображением определяемым строками ‘from’ и ‘to’. ‘from’ и ‘to’ должны быть корректными ASCII строками одного размера. Не ASCII символы в оригинальной строке не изменяются.
+
+Example:
+
+``` sql
+SELECT translate('Hello, World!', 'delor', 'DELOR') AS res
+```
+
+``` text
+┌─res───────────┐
+│ HELLO, WORLD! │
+└───────────────┘
+```
+
+## translateUTF8(string, from, to)
+
+Аналогично предыдущей функции, но работает со строками, состоящими из UTF-8 символов. ‘from’ и ‘to’ должны быть корректными UTF-8 строками одного размера.
+
+Example:
+
+``` sql
+SELECT translateUTF8('Hélló, Wórld¡', 'óé¡', 'oe!') AS res
+```
+
+``` text
+┌─res───────────┐
+│ Hello, World! │
+└───────────────┘
+```

docs/ru/sql-reference/statements/alter/partition.md

@@ -106,7 +106,8 @@ ALTER TABLE table2 [ON CLUSTER cluster] ATTACH PARTITION partition_expr FROM tab
 Следует иметь в виду:
 
 -   Таблицы должны иметь одинаковую структуру.
--   Для таблиц должен быть задан одинаковый ключ партиционирования.
+-   Для таблиц должен быть задан одинаковый ключ партиционирования, одинаковый ключ сортировки и одинаковый первичный ключ.
+-   Для таблиц должна быть задана одинаковая политика хранения (диск, на котором хранится партиция, должен быть доступен для обеих таблиц).
 
 Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr).
 
@@ -121,7 +122,8 @@ ALTER TABLE table2 [ON CLUSTER cluster] REPLACE PARTITION partition_expr FROM ta
 Следует иметь в виду:
 
 -   Таблицы должны иметь одинаковую структуру.
--   Для таблиц должен быть задан одинаковый ключ партиционирования.
+-   Для таблиц должен быть задан одинаковый ключ партиционирования, одинаковый ключ сортировки и одинаковый первичный ключ.
+-   Для таблиц должна быть задана одинаковая политика хранения (диск, на котором хранится партиция, должен быть доступен для обеих таблиц).
 
 Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr).
 
@@ -136,9 +138,9 @@ ALTER TABLE table_source [ON CLUSTER cluster] MOVE PARTITION partition_expr TO T
 Следует иметь в виду:
 
 -   Таблицы должны иметь одинаковую структуру.
--   Для таблиц должен быть задан одинаковый ключ партиционирования.
+-   Для таблиц должен быть задан одинаковый ключ партиционирования, одинаковый ключ сортировки и одинаковый первичный ключ.
+-   Для таблиц должна быть задана одинаковая политика хранения (диск, на котором хранится партиция, должен быть доступен для обеих таблиц).
 -   Движки таблиц должны быть одинакового семейства (реплицированные или нереплицированные).
--   Для таблиц должна быть задана одинаковая политика хранения.
 
 ## CLEAR COLUMN IN PARTITION {#alter_clear-column-partition}
 

programs/client/Client.cpp

@@ -106,7 +106,9 @@ void Client::processError(const String & query) const
 std::vector<String> Client::loadWarningMessages()
 {
     std::vector<String> messages;
-    connection->sendQuery(connection_parameters.timeouts, "SELECT message FROM system.warnings", "" /* query_id */,
+    connection->sendQuery(connection_parameters.timeouts,
+                          "SELECT * FROM viewIfPermitted(SELECT message FROM system.warnings ELSE null('message String'))",
+                          "" /* query_id */,
                           QueryProcessingStage::Complete,
                           &global_context->getSettingsRef(),
                           &global_context->getClientInfo(), false, {});

programs/copier/ZooKeeperStaff.h

@@ -20,7 +20,7 @@ public:
 
     bool operator<=(const WrappingUInt32 & other) const
     {
-        const UInt32 HALF = 1 << 31;
+        const UInt32 HALF = static_cast<UInt32>(1) << 31;
         return (value <= other.value && other.value - value < HALF)
                || (value > other.value && value - other.value > HALF);
     }

programs/disks/DisksApp.cpp

@@ -154,7 +154,7 @@ int DisksApp::main(const std::vector<String> & /*args*/)
         Poco::Logger::root().setLevel(Poco::Logger::parseLevel(log_level));
 
         auto log_path = config().getString("logger.clickhouse-disks", "/var/log/clickhouse-server/clickhouse-disks.log");
-        Poco::Logger::root().setChannel(new Poco::FileChannel(log_path));
+        Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::FileChannel>(new Poco::FileChannel(log_path)));
     }
 
     if (config().has("config-file") || fs::exists(getDefaultConfigFileName()))

programs/server/config.xml

@@ -604,9 +604,23 @@
              if this setting is true the user B will see all rows, and if this setting is false the user B will see no rows.
              By default this setting is false for compatibility with earlier access configurations. -->
         <users_without_row_policies_can_read_rows>false</users_without_row_policies_can_read_rows>
+        
         <!-- By default, for backward compatibility ON CLUSTER queries ignore CLUSTER grant,
              however you can change this behaviour by setting this to true -->
         <on_cluster_queries_require_cluster_grant>false</on_cluster_queries_require_cluster_grant>
+
+        <!-- By default, for backward compatibility "SELECT * FROM system.<table>" doesn't require any grants and can be executed
+             by any user. You can change this behaviour by setting this to true.
+             If it's set to true then this query requires "GRANT SELECT ON system.<table>" just like as for non-system tables.
+             Exceptions: a few system tables ("tables", "columns", "databases", and some constant tables like "one", "contributors")
+             are still accessible for everyone; and if there is a SHOW privilege (e.g. "SHOW USERS") granted the corresponding system
+             table (i.e. "system.users") will be accessible. -->
+        <select_from_system_db_requires_grant>false</select_from_system_db_requires_grant>
+
+        <!-- By default, for backward compatibility "SELECT * FROM information_schema.<table>" doesn't require any grants and can be
+             executed by any user. You can change this behaviour by setting this to true.
+             If it's set to true then this query requires "GRANT SELECT ON information_schema.<table>" just like as for ordinary tables. -->
+        <select_from_information_schema_requires_grant>false</select_from_information_schema_requires_grant>
     </access_control_improvements>
 
     <!-- Default profile of settings. -->

src/Access/AccessControl.cpp

@@ -101,7 +101,7 @@ public:
         registered_prefixes = prefixes_;
     }
 
-    bool isSettingNameAllowed(const std::string_view & setting_name) const
+    bool isSettingNameAllowed(std::string_view setting_name) const
     {
         if (Settings::hasBuiltin(setting_name))
             return true;
@@ -116,7 +116,7 @@ public:
         return false;
     }
 
-    void checkSettingNameIsAllowed(const std::string_view & setting_name) const
+    void checkSettingNameIsAllowed(std::string_view setting_name) const
     {
         if (isSettingNameAllowed(setting_name))
             return;
@@ -165,13 +165,12 @@ void AccessControl::setUpFromMainConfig(const Poco::Util::AbstractConfiguration
     setNoPasswordAllowed(config_.getBool("allow_no_password", true));
     setPlaintextPasswordAllowed(config_.getBool("allow_plaintext_password", true));
 
-    setEnabledUsersWithoutRowPoliciesCanReadRows(config_.getBool(
-        "access_control_improvements.users_without_row_policies_can_read_rows",
-        false /* false because we need to be compatible with earlier access configurations */));
-
-    setOnClusterQueriesRequireClusterGrant(config_.getBool(
-        "access_control_improvements.on_cluster_queries_require_cluster_grant",
-        false /* false because we need to be compatible with earlier access configurations */));
+    /// Optional improvements in access control system.
+    /// The default values are false because we need to be compatible with earlier access configurations
+    setEnabledUsersWithoutRowPoliciesCanReadRows(config_.getBool("access_control_improvements.users_without_row_policies_can_read_rows", false));
+    setOnClusterQueriesRequireClusterGrant(config_.getBool("access_control_improvements.on_cluster_queries_require_cluster_grant", false));
+    setSelectFromSystemDatabaseRequiresGrant(config_.getBool("access_control_improvements.select_from_system_db_requires_grant", false));
+    setSelectFromInformationSchemaRequiresGrant(config_.getBool("access_control_improvements.select_from_information_schema_requires_grant", false));
 
     addStoragesFromMainConfig(config_, config_path_, get_zookeeper_function_);
 }

src/Access/AccessControl.h

@@ -152,6 +152,12 @@ public:
     void setOnClusterQueriesRequireClusterGrant(bool enable) { on_cluster_queries_require_cluster_grant = enable; }
     bool doesOnClusterQueriesRequireClusterGrant() const { return on_cluster_queries_require_cluster_grant; }
 
+    void setSelectFromSystemDatabaseRequiresGrant(bool enable) { select_from_system_db_requires_grant = enable; }
+    bool doesSelectFromSystemDatabaseRequireGrant() const { return select_from_system_db_requires_grant; }
+
+    void setSelectFromInformationSchemaRequiresGrant(bool enable) { select_from_information_schema_requires_grant = enable; }
+    bool doesSelectFromInformationSchemaRequireGrant() const { return select_from_information_schema_requires_grant; }
+
     std::shared_ptr<const ContextAccess> getContextAccess(
         const UUID & user_id,
         const std::vector<UUID> & current_roles,
@@ -215,6 +221,8 @@ private:
     std::atomic_bool allow_no_password = true;
     std::atomic_bool users_without_row_policies_can_read_rows = false;
     std::atomic_bool on_cluster_queries_require_cluster_grant = false;
+    std::atomic_bool select_from_system_db_requires_grant = false;
+    std::atomic_bool select_from_information_schema_requires_grant = false;
 };
 
 }

src/Access/AccessRights.cpp

@@ -252,7 +252,7 @@ public:
     }
 
     template <typename ... Args>
-    void grant(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
+    void grant(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
     {
         auto & child = getChild(name);
         child.grant(flags_, subnames...);
@@ -279,7 +279,7 @@ public:
     }
 
     template <typename... Args>
-    void revoke(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
+    void revoke(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
     {
         auto & child = getChild(name);
 
@@ -306,7 +306,7 @@ public:
     }
 
     template <typename... Args>
-    bool isGranted(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames) const
+    bool isGranted(const AccessFlags & flags_, std::string_view name, const Args &... subnames) const
     {
         AccessFlags flags_to_check = flags_ - min_flags_with_children;
         if (!flags_to_check)
@@ -388,11 +388,11 @@ public:
         return res;
     }
 
-    void modifyFlags(const ModifyFlagsFunction & function, bool & flags_added, bool & flags_removed)
+    void modifyFlags(const ModifyFlagsFunction & function, bool grant_option, bool & flags_added, bool & flags_removed)
     {
         flags_added = false;
         flags_removed = false;
-        modifyFlagsRec(function, flags_added, flags_removed);
+        modifyFlagsRec(function, grant_option, flags_added, flags_removed);
         if (flags_added || flags_removed)
             optimizeTree();
     }
@@ -415,7 +415,7 @@ private:
     AccessFlags getAllGrantableFlags() const { return ::DB::getAllGrantableFlags(level); }
     AccessFlags getChildAllGrantableFlags() const { return ::DB::getAllGrantableFlags(static_cast<Level>(level + 1)); }
 
-    Node * tryGetChild(const std::string_view & name) const
+    Node * tryGetChild(std::string_view name) const
     {
         if (!children)
             return nullptr;
@@ -425,7 +425,7 @@ private:
         return &it->second;
     }
 
-    Node & getChild(const std::string_view & name)
+    Node & getChild(std::string_view name)
     {
         auto * child = tryGetChild(name);
         if (child)
@@ -669,11 +669,11 @@ private:
     }
 
     template <typename ... ParentNames>
-    void modifyFlagsRec(const ModifyFlagsFunction & function, bool & flags_added, bool & flags_removed, const ParentNames & ... parent_names)
+    void modifyFlagsRec(const ModifyFlagsFunction & function, bool grant_option, bool & flags_added, bool & flags_removed, const ParentNames & ... parent_names)
     {
-        auto invoke = [&function](const AccessFlags & flags_, const AccessFlags & min_flags_with_children_, const AccessFlags & max_flags_with_children_, std::string_view database_ = {}, std::string_view table_ = {}, std::string_view column_ = {}) -> AccessFlags
+        auto invoke = [function, grant_option](const AccessFlags & flags_, const AccessFlags & min_flags_with_children_, const AccessFlags & max_flags_with_children_, std::string_view database_ = {}, std::string_view table_ = {}, std::string_view column_ = {}) -> AccessFlags
         {
-            return function(flags_, min_flags_with_children_, max_flags_with_children_, database_, table_, column_);
+            return function(flags_, min_flags_with_children_, max_flags_with_children_, database_, table_, column_, grant_option);
         };
 
         if constexpr (sizeof...(ParentNames) < 3)
@@ -683,7 +683,7 @@ private:
                 for (auto & child : *children | boost::adaptors::map_values)
                 {
                     const String & child_name = *child.node_name;
-                    child.modifyFlagsRec(function, flags_added, flags_removed, parent_names..., child_name);
+                    child.modifyFlagsRec(function, grant_option, flags_added, flags_removed, parent_names..., child_name);
                 }
             }
         }
@@ -819,20 +819,20 @@ void AccessRights::grantImpl(const AccessRightsElements & elements)
 }
 
 void AccessRights::grant(const AccessFlags & flags) { grantImpl<false>(flags); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl<false>(flags, database); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<false>(flags, database, table); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<false>(flags, database, table, column); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database) { grantImpl<false>(flags, database); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<false>(flags, database, table); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<false>(flags, database, table, column); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
 void AccessRights::grant(const AccessRightsElement & element) { grantImpl<false>(element); }
 void AccessRights::grant(const AccessRightsElements & elements) { grantImpl<false>(elements); }
 
 void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl<true>(flags); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database) { grantImpl<true>(flags, database); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<true>(flags, database, table); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<true>(flags, database, table, column); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database) { grantImpl<true>(flags, database); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<true>(flags, database, table); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<true>(flags, database, table, column); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
 void AccessRights::grantWithGrantOption(const AccessRightsElement & element) { grantImpl<true>(element); }
 void AccessRights::grantWithGrantOption(const AccessRightsElements & elements) { grantImpl<true>(elements); }
 
@@ -892,20 +892,20 @@ void AccessRights::revokeImpl(const AccessRightsElements & elements)
 }
 
 void AccessRights::revoke(const AccessFlags & flags) { revokeImpl<false>(flags); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl<false>(flags, database); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<false>(flags, database, table); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<false>(flags, database, table, column); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database) { revokeImpl<false>(flags, database); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<false>(flags, database, table); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<false>(flags, database, table, column); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
 void AccessRights::revoke(const AccessRightsElement & element) { revokeImpl<false>(element); }
 void AccessRights::revoke(const AccessRightsElements & elements) { revokeImpl<false>(elements); }
 
 void AccessRights::revokeGrantOption(const AccessFlags & flags) { revokeImpl<true>(flags); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database) { revokeImpl<true>(flags, database); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<true>(flags, database, table); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<true>(flags, database, table, column); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database) { revokeImpl<true>(flags, database); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<true>(flags, database, table); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<true>(flags, database, table, column); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
 void AccessRights::revokeGrantOption(const AccessRightsElement & element) { revokeImpl<true>(element); }
 void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { revokeImpl<true>(elements); }
 
@@ -984,20 +984,20 @@ bool AccessRights::isGrantedImpl(const AccessRightsElements & elements) const
 }
 
 bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl<false>(flags); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<false>(flags, database); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<false>(flags, database, table); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<false>(flags, database, table, column); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<false>(flags, database); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<false>(flags, database, table); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<false>(flags, database, table, column); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
 bool AccessRights::isGranted(const AccessRightsElement & element) const { return isGrantedImpl<false>(element); }
 bool AccessRights::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl<false>(elements); }
 
 bool AccessRights::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl<true>(flags); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<true>(flags, database); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<true>(flags, database, table); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<true>(flags, database, table, column); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<true>(flags, database); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<true>(flags, database, table); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<true>(flags, database, table, column); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
 bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl<true>(element); }
 bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl<true>(elements); }
 
@@ -1062,24 +1062,21 @@ void AccessRights::modifyFlags(const ModifyFlagsFunction & function)
 {
     if (!root)
         return;
+
     bool flags_added, flags_removed;
-    root->modifyFlags(function, flags_added, flags_removed);
+    root->modifyFlags(function, false, flags_added, flags_removed);
     if (flags_removed && root_with_grant_option)
         root_with_grant_option->makeIntersection(*root);
-}
 
-
-void AccessRights::modifyFlagsWithGrantOption(const ModifyFlagsFunction & function)
-{
-    if (!root_with_grant_option)
-        return;
-    bool flags_added, flags_removed;
-    root_with_grant_option->modifyFlags(function, flags_added, flags_removed);
-    if (flags_added)
+    if (root_with_grant_option)
     {
-        if (!root)
-            root = std::make_unique<Node>();
-        root->makeUnion(*root_with_grant_option);
+        root_with_grant_option->modifyFlags(function, true, flags_added, flags_removed);
+        if (flags_added)
+        {
+            if (!root)
+                root = std::make_unique<Node>();
+            root->makeUnion(*root_with_grant_option);
+        }
     }
 }
 

src/Access/AccessRights.h

@@ -39,59 +39,59 @@ public:
     /// Grants access on a specified database/table/column.
     /// Does nothing if the specified access has been already granted.
     void grant(const AccessFlags & flags);
-    void grant(const AccessFlags & flags, const std::string_view & database);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void grant(const AccessFlags & flags, std::string_view database);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void grant(const AccessRightsElement & element);
     void grant(const AccessRightsElements & elements);
 
     void grantWithGrantOption(const AccessFlags & flags);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void grantWithGrantOption(const AccessRightsElement & element);
     void grantWithGrantOption(const AccessRightsElements & elements);
 
     /// Revokes a specified access granted earlier on a specified database/table/column.
     /// For example, revoke(AccessType::ALL) revokes all grants at all, just like clear();
     void revoke(const AccessFlags & flags);
-    void revoke(const AccessFlags & flags, const std::string_view & database);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void revoke(const AccessFlags & flags, std::string_view database);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void revoke(const AccessRightsElement & element);
     void revoke(const AccessRightsElements & elements);
 
     void revokeGrantOption(const AccessFlags & flags);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void revokeGrantOption(const AccessRightsElement & element);
     void revokeGrantOption(const AccessRightsElements & elements);
 
     /// Whether a specified access granted.
     bool isGranted(const AccessFlags & flags) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool isGranted(const AccessRightsElement & element) const;
     bool isGranted(const AccessRightsElements & elements) const;
 
     bool hasGrantOption(const AccessFlags & flags) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool hasGrantOption(const AccessRightsElement & element) const;
     bool hasGrantOption(const AccessRightsElements & elements) const;
 
@@ -107,11 +107,11 @@ public:
         const AccessFlags & flags,
         const AccessFlags & min_flags_with_children,
         const AccessFlags & max_flags_with_children,
-        const std::string_view & database,
-        const std::string_view & table,
-        const std::string_view & column)>;
+        std::string_view database,
+        std::string_view table,
+        std::string_view column,
+        bool grant_option)>;
     void modifyFlags(const ModifyFlagsFunction & function);
-    void modifyFlagsWithGrantOption(const ModifyFlagsFunction & function);
 
     friend bool operator ==(const AccessRights & left, const AccessRights & right);
     friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); }

src/Access/Authentication.cpp

@@ -26,17 +26,17 @@ namespace
         return (Util::stringToDigest(password) == password_plaintext);
     }
 
-    bool checkPasswordDoubleSHA1(const std::string_view & password, const Digest & password_double_sha1)
+    bool checkPasswordDoubleSHA1(std::string_view password, const Digest & password_double_sha1)
     {
         return (Util::encodeDoubleSHA1(password) == password_double_sha1);
     }
 
-    bool checkPasswordSHA256(const std::string_view & password, const Digest & password_sha256, const String & salt)
+    bool checkPasswordSHA256(std::string_view password, const Digest & password_sha256, const String & salt)
     {
         return Util::encodeSHA256(String(password).append(salt)) == password_sha256;
     }
 
-    bool checkPasswordDoubleSHA1MySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_double_sha1)
+    bool checkPasswordDoubleSHA1MySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_double_sha1)
     {
         /// scrambled_password = SHA1(password) XOR SHA1(scramble <concat> SHA1(SHA1(password)))
 
@@ -61,7 +61,7 @@ namespace
         return calculated_password_double_sha1 == password_double_sha1;
     }
 
-    bool checkPasswordPlainTextMySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_plaintext)
+    bool checkPasswordPlainTextMySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_plaintext)
     {
         return checkPasswordDoubleSHA1MySQL(scramble, scrambled_password, Util::encodeDoubleSHA1(password_plaintext));
     }

src/Access/Common/AccessFlags.cpp

@@ -35,7 +35,7 @@ namespace
             return access_type_to_flags_mapping[static_cast<size_t>(type)];
         }
 
-        Flags keywordToFlags(const std::string_view & keyword) const
+        Flags keywordToFlags(std::string_view keyword) const
         {
             auto it = keyword_to_flags_map.find(keyword);
             if (it == keyword_to_flags_map.end())
@@ -142,14 +142,14 @@ namespace
             }
         };
 
-        static String replaceUnderscoreWithSpace(const std::string_view & str)
+        static String replaceUnderscoreWithSpace(std::string_view str)
         {
             String res{str};
             boost::replace_all(res, "_", " ");
             return res;
         }
 
-        static Strings splitAliases(const std::string_view & str)
+        static Strings splitAliases(std::string_view str)
         {
             Strings aliases;
             boost::split(aliases, str, boost::is_any_of(","));
@@ -160,10 +160,10 @@ namespace
 
         static void makeNode(
             AccessType access_type,
-            const std::string_view & name,
-            const std::string_view & aliases,
+            std::string_view name,
+            std::string_view aliases,
             NodeType node_type,
-            const std::string_view & parent_group_name,
+            std::string_view parent_group_name,
             std::unordered_map<std::string_view, Node *> & nodes,
             std::unordered_map<std::string_view, NodePtr> & owned_nodes,
             size_t & next_flag)
@@ -353,7 +353,7 @@ namespace
 
 
 AccessFlags::AccessFlags(AccessType type) : flags(Helper::instance().accessTypeToFlags(type)) {}
-AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
+AccessFlags::AccessFlags(std::string_view keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
 AccessFlags::AccessFlags(const std::vector<std::string_view> & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
 AccessFlags::AccessFlags(const Strings & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
 String AccessFlags::toString() const { return Helper::instance().flagsToString(flags); }

src/Access/Common/AccessFlags.h

@@ -21,7 +21,7 @@ public:
     AccessFlags() = default;
 
     /// Constructs from a string like "SELECT".
-    AccessFlags(const std::string_view & keyword); /// NOLINT
+    AccessFlags(std::string_view keyword); /// NOLINT
 
     /// Constructs from a list of strings like "SELECT, UPDATE, INSERT".
     AccessFlags(const std::vector<std::string_view> & keywords); /// NOLINT

src/Access/Common/AccessRightsElement.cpp

@@ -81,7 +81,7 @@ namespace
         }
 
         bool need_comma = false;
-        for (const std::string_view & keyword : keywords)
+        for (std::string_view keyword : keywords)
         {
             if (need_comma)
                 result.append(", ");
@@ -145,18 +145,18 @@ namespace
 }
 
 
-AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_)
+AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_)
     : access_flags(access_flags_), database(database_), any_database(false)
 {
 }
 
-AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_)
+AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_)
     : access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false)
 {
 }
 
 AccessRightsElement::AccessRightsElement(
-    AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_)
+    AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_)
     : access_flags(access_flags_)
     , database(database_)
     , table(table_)
@@ -169,8 +169,8 @@ AccessRightsElement::AccessRightsElement(
 
 AccessRightsElement::AccessRightsElement(
     AccessFlags access_flags_,
-    const std::string_view & database_,
-    const std::string_view & table_,
+    std::string_view database_,
+    std::string_view table_,
     const std::vector<std::string_view> & columns_)
     : access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false), any_column(false)
 {
@@ -180,7 +180,7 @@ AccessRightsElement::AccessRightsElement(
 }
 
 AccessRightsElement::AccessRightsElement(
-    AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_)
+    AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_)
     : access_flags(access_flags_)
     , database(database_)
     , table(table_)

src/Access/Common/AccessRightsElement.h

@@ -28,19 +28,19 @@ struct AccessRightsElement
 
     explicit AccessRightsElement(AccessFlags access_flags_) : access_flags(access_flags_) {}
 
-    AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_);
-    AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_);
+    AccessRightsElement(AccessFlags access_flags_, std::string_view database_);
+    AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_);
     AccessRightsElement(
-        AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_);
+        AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_);
 
     AccessRightsElement(
         AccessFlags access_flags_,
-        const std::string_view & database_,
-        const std::string_view & table_,
+        std::string_view database_,
+        std::string_view table_,
         const std::vector<std::string_view> & columns_);
 
     AccessRightsElement(
-        AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_);
+        AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_);
 
     bool empty() const { return !access_flags || (!any_column && columns.empty()); }
 

src/Access/Common/AccessType.cpp

@@ -35,7 +35,7 @@ namespace
 #undef ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING
         }
 
-        void addToMapping(AccessType type, const std::string_view & str)
+        void addToMapping(AccessType type, std::string_view str)
         {
             String str2{str};
             boost::replace_all(str2, "_", " ");

src/Access/Common/AuthenticationData.cpp

@@ -71,7 +71,7 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty
 }
 
 
-AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::string_view & text [[maybe_unused]])
+AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_view text [[maybe_unused]])
 {
 #if USE_SSL
     Digest hash;
@@ -86,7 +86,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::str
 }
 
 
-AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(const std::string_view & text)
+AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(std::string_view text)
 {
     Poco::SHA1Engine engine;
     engine.update(text.data(), text.size());

src/Access/Common/AuthenticationData.h

@@ -96,11 +96,11 @@ public:
 
     struct Util
     {
-        static Digest stringToDigest(const std::string_view & text) { return Digest(text.data(), text.data() + text.size()); }
-        static Digest encodeSHA256(const std::string_view & text);
-        static Digest encodeSHA1(const std::string_view & text);
+        static Digest stringToDigest(std::string_view text) { return Digest(text.data(), text.data() + text.size()); }
+        static Digest encodeSHA256(std::string_view text);
+        static Digest encodeSHA1(std::string_view text);
         static Digest encodeSHA1(const Digest & text) { return encodeSHA1(std::string_view{reinterpret_cast<const char *>(text.data()), text.size()}); }
-        static Digest encodeDoubleSHA1(const std::string_view & text) { return encodeSHA1(encodeSHA1(text)); }
+        static Digest encodeDoubleSHA1(std::string_view text) { return encodeSHA1(encodeSHA1(text)); }
         static Digest encodeDoubleSHA1(const Digest & text) { return encodeSHA1(encodeSHA1(text)); }
     };
 

src/Access/ContextAccess.cpp

@@ -44,9 +44,17 @@ namespace
     }
 
 
-    AccessRights addImplicitAccessRights(const AccessRights & access)
+    AccessRights addImplicitAccessRights(const AccessRights & access, const AccessControl & access_control)
     {
-        auto modifier = [&](const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, const std::string_view & database, const std::string_view & table, const std::string_view & column) -> AccessFlags
+        AccessFlags max_flags;
+
+        auto modifier = [&](const AccessFlags & flags,
+                            const AccessFlags & min_flags_with_children,
+                            const AccessFlags & max_flags_with_children,
+                            std::string_view database,
+                            std::string_view table,
+                            std::string_view column,
+                            bool /* grant_option */) -> AccessFlags
         {
             size_t level = !database.empty() + !table.empty() + !column.empty();
             AccessFlags res = flags;
@@ -115,17 +123,80 @@ namespace
                 res |= show_databases;
             }
 
+            max_flags |= res;
+
             return res;
         };
 
         AccessRights res = access;
         res.modifyFlags(modifier);
-        res.modifyFlagsWithGrantOption(modifier);
 
-        /// Anyone has access to the "system" and "information_schema" database.
-        res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
-        res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA);
-        res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE);
+        /// If "select_from_system_db_requires_grant" is enabled we provide implicit grants only for a few tables in the system database.
+        if (access_control.doesSelectFromSystemDatabaseRequireGrant())
+        {
+            const char * always_accessible_tables[] = {
+                /// Constant tables
+                "one",
+
+                /// "numbers", "numbers_mt", "zeros", "zeros_mt" were excluded because they can generate lots of values and
+                /// that can decrease performance in some cases.
+
+                "contributors",
+                "licenses",
+                "time_zones",
+                "collations",
+
+                "formats",
+                "privileges",
+                "data_type_families",
+                "table_engines",
+                "table_functions",
+                "aggregate_function_combinators",
+
+                "functions", /// Can contain user-defined functions
+
+                /// The following tables hide some rows if the current user doesn't have corresponding SHOW privileges.
+                "databases",
+                "tables",
+                "columns",
+
+                /// Specific to the current session
+                "settings",
+                "current_roles",
+                "enabled_roles",
+                "quota_usage"
+            };
+
+            for (const auto * table_name : always_accessible_tables)
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, table_name);
+
+            if (max_flags.contains(AccessType::SHOW_USERS))
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "users");
+
+            if (max_flags.contains(AccessType::SHOW_ROLES))
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "roles");
+
+            if (max_flags.contains(AccessType::SHOW_ROW_POLICIES))
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "row_policies");
+
+            if (max_flags.contains(AccessType::SHOW_SETTINGS_PROFILES))
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "settings_profiles");
+
+            if (max_flags.contains(AccessType::SHOW_QUOTAS))
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "quotas");
+        }
+        else
+        {
+            res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
+        }
+
+        /// If "select_from_information_schema_requires_grant" is enabled we don't provide implicit grants for the information_schema database.
+        if (!access_control.doesSelectFromInformationSchemaRequireGrant())
+        {
+            res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA);
+            res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE);
+        }
+
         return res;
     }
 
@@ -141,7 +212,7 @@ namespace
     std::string_view getDatabase() { return {}; }
 
     template <typename... OtherArgs>
-    std::string_view getDatabase(const std::string_view & arg1, const OtherArgs &...) { return arg1; }
+    std::string_view getDatabase(std::string_view arg1, const OtherArgs &...) { return arg1; }
 }
 
 
@@ -247,7 +318,7 @@ void ContextAccess::setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> &
 void ContextAccess::calculateAccessRights() const
 {
     access = std::make_shared<AccessRights>(mixAccessRightsFromUserAndRoles(*user, *roles_info));
-    access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*access));
+    access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*access, *access_control));
 
     if (trace_log)
     {
@@ -342,7 +413,7 @@ std::shared_ptr<const ContextAccess> ContextAccess::getFullAccess()
         auto full_access = std::shared_ptr<ContextAccess>(new ContextAccess);
         full_access->is_full_access = true;
         full_access->access = std::make_shared<AccessRights>(AccessRights::getFullAccess());
-        full_access->access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*full_access->access));
+        full_access->access_with_implicit = full_access->access;
         return full_access;
     }();
     return res;
@@ -413,7 +484,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
     };
 
     if (is_full_access)
-        return access_granted();
+        return true;
 
     if (user_was_dropped)
         return access_denied("User has been dropped", ErrorCodes::UNKNOWN_USER);
@@ -422,7 +493,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
         flags &= ~AccessType::CLUSTER;
 
     if (!flags)
-        return access_granted();
+        return true;
 
     /// Access to temporary tables is controlled in an unusual way, not like normal tables.
     /// Creating of temporary tables is controlled by AccessType::CREATE_TEMPORARY_TABLES grant,
@@ -519,7 +590,7 @@ bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const
 }
 
 template <bool throw_if_denied, bool grant_option, typename... Args>
-bool ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const
+bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const
 {
     return checkAccessImplHelper<throw_if_denied, grant_option>(flags, database.empty() ? params.current_database : database, args...);
 }
@@ -564,38 +635,38 @@ bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
 }
 
 bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl<false, false>(flags); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, false>(flags, database); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, false>(flags, database, table); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(flags, database); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(flags, database, table); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
 bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl<false, false>(element); }
 bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(elements); }
 
 bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return checkAccessImpl<false, true>(flags); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, true>(flags, database); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, true>(flags, database, table); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(flags, database); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(flags, database, table); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
 bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return checkAccessImpl<false, true>(element); }
 bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(elements); }
 
 void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<true, false>(flags); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, false>(flags, database); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, false>(flags, database, table); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, false>(flags, database, table, column); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(flags, database); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(flags, database, table); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(flags, database, table, column); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
 void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<true, false>(element); }
 void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<true, false>(elements); }
 
 void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<true, true>(flags); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, true>(flags, database); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, true>(flags, database, table); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, true>(flags, database, table, column); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(flags, database); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(flags, database, table); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(flags, database, table, column); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
 void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<true, true>(element); }
 void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true, true>(elements); }
 

src/Access/ContextAccess.h

@@ -101,40 +101,40 @@ public:
     /// Checks if a specified access is granted, and throws an exception if not.
     /// Empty database means the current database.
     void checkAccess(const AccessFlags & flags) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     void checkAccess(const AccessRightsElement & element) const;
     void checkAccess(const AccessRightsElements & elements) const;
 
     void checkGrantOption(const AccessFlags & flags) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     void checkGrantOption(const AccessRightsElement & element) const;
     void checkGrantOption(const AccessRightsElements & elements) const;
 
     /// Checks if a specified access is granted, and returns false if not.
     /// Empty database means the current database.
     bool isGranted(const AccessFlags & flags) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool isGranted(const AccessRightsElement & element) const;
     bool isGranted(const AccessRightsElements & elements) const;
 
     bool hasGrantOption(const AccessFlags & flags) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool hasGrantOption(const AccessRightsElement & element) const;
     bool hasGrantOption(const AccessRightsElements & elements) const;
 
@@ -180,7 +180,7 @@ private:
     bool checkAccessImpl(const AccessFlags & flags) const;
 
     template <bool throw_if_denied, bool grant_option, typename... Args>
-    bool checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const;
+    bool checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const;
 
     template <bool throw_if_denied, bool grant_option>
     bool checkAccessImpl(const AccessRightsElement & element) const;

src/Access/SettingsConstraints.cpp

@@ -36,12 +36,12 @@ void SettingsConstraints::clear()
 }
 
 
-void SettingsConstraints::setMinValue(const std::string_view & setting_name, const Field & min_value)
+void SettingsConstraints::setMinValue(std::string_view setting_name, const Field & min_value)
 {
     getConstraintRef(setting_name).min_value = Settings::castValueUtil(setting_name, min_value);
 }
 
-Field SettingsConstraints::getMinValue(const std::string_view & setting_name) const
+Field SettingsConstraints::getMinValue(std::string_view setting_name) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -51,12 +51,12 @@ Field SettingsConstraints::getMinValue(const std::string_view & setting_name) co
 }
 
 
-void SettingsConstraints::setMaxValue(const std::string_view & setting_name, const Field & max_value)
+void SettingsConstraints::setMaxValue(std::string_view setting_name, const Field & max_value)
 {
     getConstraintRef(setting_name).max_value = Settings::castValueUtil(setting_name, max_value);
 }
 
-Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) const
+Field SettingsConstraints::getMaxValue(std::string_view setting_name) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -66,12 +66,12 @@ Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) co
 }
 
 
-void SettingsConstraints::setReadOnly(const std::string_view & setting_name, bool read_only)
+void SettingsConstraints::setReadOnly(std::string_view setting_name, bool read_only)
 {
     getConstraintRef(setting_name).read_only = read_only;
 }
 
-bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) const
+bool SettingsConstraints::isReadOnly(std::string_view setting_name) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -81,7 +81,7 @@ bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) cons
 }
 
 
-void SettingsConstraints::set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only)
+void SettingsConstraints::set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only)
 {
     auto & ref = getConstraintRef(setting_name);
     ref.min_value = Settings::castValueUtil(setting_name, min_value);
@@ -89,7 +89,7 @@ void SettingsConstraints::set(const std::string_view & setting_name, const Field
     ref.read_only = read_only;
 }
 
-void SettingsConstraints::get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const
+void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -318,7 +318,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh
 }
 
 
-SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const std::string_view & setting_name)
+SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(std::string_view setting_name)
 {
     auto it = constraints.find(setting_name);
     if (it == constraints.end())
@@ -331,7 +331,7 @@ SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const st
     return it->second;
 }
 
-const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(const std::string_view & setting_name) const
+const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(std::string_view setting_name) const
 {
     auto it = constraints.find(setting_name);
     if (it == constraints.end())

src/Access/SettingsConstraints.h

@@ -61,17 +61,17 @@ public:
     void clear();
     bool empty() const { return constraints.empty(); }
 
-    void setMinValue(const std::string_view & setting_name, const Field & min_value);
-    Field getMinValue(const std::string_view & setting_name) const;
+    void setMinValue(std::string_view setting_name, const Field & min_value);
+    Field getMinValue(std::string_view setting_name) const;
 
-    void setMaxValue(const std::string_view & setting_name, const Field & max_value);
-    Field getMaxValue(const std::string_view & setting_name) const;
+    void setMaxValue(std::string_view setting_name, const Field & max_value);
+    Field getMaxValue(std::string_view setting_name) const;
 
-    void setReadOnly(const std::string_view & setting_name, bool read_only);
-    bool isReadOnly(const std::string_view & setting_name) const;
+    void setReadOnly(std::string_view setting_name, bool read_only);
+    bool isReadOnly(std::string_view setting_name) const;
 
-    void set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only);
-    void get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const;
+    void set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only);
+    void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const;
 
     void merge(const SettingsConstraints & other);
 
@@ -105,8 +105,8 @@ private:
     };
     bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const;
 
-    Constraint & getConstraintRef(const std::string_view & setting_name);
-    const Constraint * tryGetConstraint(const std::string_view & setting_name) const;
+    Constraint & getConstraintRef(std::string_view setting_name);
+    const Constraint * tryGetConstraint(std::string_view setting_name) const;
 
     std::unordered_map<std::string_view, Constraint> constraints;
     const AccessControl * access_control = nullptr;

src/AggregateFunctions/AggregateFunctionMap.h

@@ -132,7 +132,7 @@ public:
                     key_ref = assert_cast<const ColumnString &>(key_column).getDataAt(offset + i);
 
 #ifdef __cpp_lib_generic_unordered_lookup
-                key = static_cast<std::string_view>(key_ref);
+                key = key_ref.toView();
 #else
                 key = key_ref.toString();
 #endif

src/AggregateFunctions/ThetaSketchData.h

@@ -43,7 +43,7 @@ public:
     ~ThetaSketchData() = default;
 
     /// Insert original value without hash, as `datasketches::update_theta_sketch.update` will do the hash internal.
-    void insertOriginal(const StringRef & value)
+    void insertOriginal(StringRef value)
     {
         getSkUpdate()->update(value.data, value.size);
     }

src/Backups/BackupFactory.h

@@ -25,7 +25,6 @@ public:
     struct CreateParams
     {
         OpenMode open_mode = OpenMode::WRITE;
-        std::optional<UUID> backup_uuid;
         BackupInfo backup_info;
         std::optional<BackupInfo> base_backup_info;
         String compression_method;
@@ -34,6 +33,7 @@ public:
         ContextPtr context;
         bool is_internal_backup = false;
         std::shared_ptr<IBackupCoordination> backup_coordination;
+        std::optional<UUID> backup_uuid;
     };
 
     static BackupFactory & instance();

src/Backups/BackupIO.h

@@ -23,8 +23,9 @@ class IBackupWriter /// BackupWriterFile, BackupWriterDisk, BackupWriterS3
 public:
     virtual ~IBackupWriter() = default;
     virtual bool fileExists(const String & file_name) = 0;
+    virtual bool fileContentsEqual(const String & file_name, const String & expected_file_contents) = 0;
     virtual std::unique_ptr<WriteBuffer> writeFile(const String & file_name) = 0;
-    virtual void removeFilesAfterFailure(const Strings & file_names) = 0;
+    virtual void removeFiles(const Strings & file_names) = 0;
 };
 
 }

src/Backups/BackupIO_Disk.cpp

@@ -38,6 +38,25 @@ bool BackupWriterDisk::fileExists(const String & file_name)
     return disk->exists(path / file_name);
 }
 
+bool BackupWriterDisk::fileContentsEqual(const String & file_name, const String & expected_file_contents)
+{
+    if (!disk->exists(path / file_name))
+        return false;
+
+    try
+    {
+        auto in = disk->readFile(path / file_name);
+        String actual_file_contents(expected_file_contents.size(), ' ');
+        return (in->read(actual_file_contents.data(), actual_file_contents.size()) == actual_file_contents.size())
+            && (actual_file_contents == expected_file_contents) && in->eof();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+        return false;
+    }
+}
+
 std::unique_ptr<WriteBuffer> BackupWriterDisk::writeFile(const String & file_name)
 {
     auto file_path = path / file_name;
@@ -45,7 +64,7 @@ std::unique_ptr<WriteBuffer> BackupWriterDisk::writeFile(const String & file_nam
     return disk->writeFile(file_path);
 }
 
-void BackupWriterDisk::removeFilesAfterFailure(const Strings & file_names)
+void BackupWriterDisk::removeFiles(const Strings & file_names)
 {
     for (const auto & file_name : file_names)
         disk->removeFileIfExists(path / file_name);

src/Backups/BackupIO_Disk.h

@@ -30,8 +30,9 @@ public:
     ~BackupWriterDisk() override;
 
     bool fileExists(const String & file_name) override;
+    bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
-    void removeFilesAfterFailure(const Strings & file_names) override;
+    void removeFiles(const Strings & file_names) override;
 
 private:
     DiskPtr disk;

src/Backups/BackupIO_File.cpp

@@ -39,6 +39,25 @@ bool BackupWriterFile::fileExists(const String & file_name)
     return fs::exists(path / file_name);
 }
 
+bool BackupWriterFile::fileContentsEqual(const String & file_name, const String & expected_file_contents)
+{
+    if (!fs::exists(path / file_name))
+        return false;
+
+    try
+    {
+        auto in = createReadBufferFromFileBase(path / file_name, {});
+        String actual_file_contents(expected_file_contents.size(), ' ');
+        return (in->read(actual_file_contents.data(), actual_file_contents.size()) == actual_file_contents.size())
+            && (actual_file_contents == expected_file_contents) && in->eof();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+        return false;
+    }
+}
+
 std::unique_ptr<WriteBuffer> BackupWriterFile::writeFile(const String & file_name)
 {
     auto file_path = path / file_name;
@@ -46,7 +65,7 @@ std::unique_ptr<WriteBuffer> BackupWriterFile::writeFile(const String & file_nam
     return std::make_unique<WriteBufferFromFile>(file_path);
 }
 
-void BackupWriterFile::removeFilesAfterFailure(const Strings & file_names)
+void BackupWriterFile::removeFiles(const Strings & file_names)
 {
     for (const auto & file_name : file_names)
         fs::remove(path / file_name);

src/Backups/BackupIO_File.h

@@ -27,8 +27,9 @@ public:
     ~BackupWriterFile() override;
 
     bool fileExists(const String & file_name) override;
+    bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
     std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
-    void removeFilesAfterFailure(const Strings & file_names) override;
+    void removeFiles(const Strings & file_names) override;
 
 private:
     std::filesystem::path path;

src/Backups/BackupImpl.cpp

@@ -37,6 +37,7 @@ namespace ErrorCodes
     extern const int BACKUP_ENTRY_ALREADY_EXISTS;
     extern const int BACKUP_ENTRY_NOT_FOUND;
     extern const int BACKUP_IS_EMPTY;
+    extern const int FAILED_TO_SYNC_BACKUP_OR_RESTORE;
     extern const int LOGICAL_ERROR;
 }
 
@@ -146,9 +147,9 @@ BackupImpl::BackupImpl(
     const std::optional<BackupInfo> & base_backup_info_,
     std::shared_ptr<IBackupWriter> writer_,
     const ContextPtr & context_,
-    const std::optional<UUID> & backup_uuid_,
     bool is_internal_backup_,
-    const std::shared_ptr<IBackupCoordination> & coordination_)
+    const std::shared_ptr<IBackupCoordination> & coordination_,
+    const std::optional<UUID> & backup_uuid_)
     : backup_name(backup_name_)
     , archive_params(archive_params_)
     , use_archives(!archive_params.archive_name.empty())
@@ -177,42 +178,28 @@ BackupImpl::~BackupImpl()
     }
 }
 
-
 void BackupImpl::open(const ContextPtr & context)
 {
     std::lock_guard lock{mutex};
 
-    String file_name_to_check_existence;
-    if (use_archives)
-        file_name_to_check_existence = archive_params.archive_name;
-    else
-        file_name_to_check_existence = ".backup";
-    bool backup_exists = (open_mode == OpenMode::WRITE) ? writer->fileExists(file_name_to_check_existence) : reader->fileExists(file_name_to_check_existence);
-
-    if (open_mode == OpenMode::WRITE)
-    {
-        if (backup_exists)
-            throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "Backup {} already exists", backup_name);
-    }
-    else
-    {
-        if (!backup_exists)
-            throw Exception(ErrorCodes::BACKUP_NOT_FOUND, "Backup {} not found", backup_name);
-    }
-
     if (open_mode == OpenMode::WRITE)
     {
         timestamp = std::time(nullptr);
         if (!uuid)
             uuid = UUIDHelpers::generateV4();
+        lock_file_name = use_archives ? (archive_params.archive_name + ".lock") : ".lock";
         writing_finalized = false;
+
+        /// Check that we can write a backup there and create the lock file to own this destination.
+        checkBackupDoesntExist();
+        if (!is_internal_backup)
+            createLockFile();
+        checkLockFile(true);
     }
 
     if (open_mode == OpenMode::READ)
         readBackupMetadata();
 
-    assert(uuid); /// Backup's UUID must be loaded or generated at this point.
-
     if (base_backup_info)
     {
         BackupFactory::CreateParams params;
@@ -253,6 +240,8 @@ time_t BackupImpl::getTimestamp() const
 
 void BackupImpl::writeBackupMetadata()
 {
+    assert(!is_internal_backup);
+
     Poco::AutoPtr<Poco::Util::XMLConfiguration> config{new Poco::Util::XMLConfiguration()};
     config->setUInt("version", CURRENT_BACKUP_VERSION);
     config->setString("timestamp", toString(LocalDateTime{timestamp}));
@@ -308,6 +297,8 @@ void BackupImpl::writeBackupMetadata()
     config->save(stream);
     String str = stream.str();
 
+    checkLockFile(true);
+
     std::unique_ptr<WriteBuffer> out;
     if (use_archives)
         out = getArchiveWriter("")->writeFile(".backup");
@@ -321,9 +312,17 @@ void BackupImpl::readBackupMetadata()
 {
     std::unique_ptr<ReadBuffer> in;
     if (use_archives)
+    {
+        if (!reader->fileExists(archive_params.archive_name))
+            throw Exception(ErrorCodes::BACKUP_NOT_FOUND, "Backup {} not found", backup_name);
         in = getArchiveReader("")->readFile(".backup");
+    }
     else
+    {
+        if (!reader->fileExists(".backup"))
+            throw Exception(ErrorCodes::BACKUP_NOT_FOUND, "Backup {} not found", backup_name);
         in = reader->readFile(".backup");
+    }
 
     String str;
     readStringUntilEOF(str, *in);
@@ -387,6 +386,59 @@ void BackupImpl::readBackupMetadata()
     }
 }
 
+void BackupImpl::checkBackupDoesntExist() const
+{
+    String file_name_to_check_existence;
+    if (use_archives)
+        file_name_to_check_existence = archive_params.archive_name;
+    else
+        file_name_to_check_existence = ".backup";
+
+    if (writer->fileExists(file_name_to_check_existence))
+        throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "Backup {} already exists", backup_name);
+
+    /// Check that no other backup (excluding internal backups) is writing to the same destination.
+    if (!is_internal_backup)
+    {
+        assert(!lock_file_name.empty());
+        if (writer->fileExists(lock_file_name))
+            throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "Backup {} is being written already", backup_name);
+    }
+}
+
+void BackupImpl::createLockFile()
+{
+    /// Internal backup must not create the lock file (it should be created by the initiator).
+    assert(!is_internal_backup);
+
+    assert(uuid);
+    auto out = writer->writeFile(lock_file_name);
+    writeUUIDText(*uuid, *out);
+}
+
+bool BackupImpl::checkLockFile(bool throw_if_failed) const
+{
+    if (!lock_file_name.empty() && uuid && writer->fileContentsEqual(lock_file_name, toString(*uuid)))
+        return true;
+
+    if (throw_if_failed)
+    {
+        if (!writer->fileExists(lock_file_name))
+            throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "Lock file {} suddenly disappeared while writing backup {}", lock_file_name, backup_name);
+        throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "A concurrent backup writing to the same destination {} detected", backup_name);
+    }
+    return false;
+}
+
+void BackupImpl::removeLockFile()
+{
+    if (is_internal_backup)
+        return; /// Internal backup must not remove the lock file (it's still used by the initiator).
+
+    if (checkLockFile(false))
+        writer->removeFiles({lock_file_name});
+}
+
 Strings BackupImpl::listFiles(const String & directory, bool recursive) const
 {
     std::lock_guard lock{mutex};
@@ -648,6 +700,9 @@ void BackupImpl::writeFile(const String & file_name, BackupEntryPtr entry)
         read_buffer = entry->getReadBuffer();
     read_buffer->seek(copy_pos, SEEK_SET);
 
+    if (!num_files_written)
+        checkLockFile(true);
+
     /// Copy the entry's data after `copy_pos`.
     std::unique_ptr<WriteBuffer> out;
     if (use_archives)
@@ -675,6 +730,7 @@ void BackupImpl::writeFile(const String & file_name, BackupEntryPtr entry)
 
     copyData(*read_buffer, *out);
     out->finalize();
+    ++num_files_written;
 }
 
 
@@ -694,6 +750,7 @@ void BackupImpl::finalizeWriting()
     {
         LOG_TRACE(log, "Finalizing backup {}", backup_name);
         writeBackupMetadata();
+        removeLockFile();
         LOG_TRACE(log, "Finalized backup {}", backup_name);
     }
 
@@ -741,6 +798,9 @@ std::shared_ptr<IArchiveWriter> BackupImpl::getArchiveWriter(const String & suff
 
 void BackupImpl::removeAllFilesAfterFailure()
 {
+    if (is_internal_backup)
+        return; /// Let the initiator remove unnecessary files.
+
     try
     {
         LOG_INFO(log, "Removing all files of backup {} after failure", backup_name);
@@ -762,7 +822,11 @@ void BackupImpl::removeAllFilesAfterFailure()
                 files_to_remove.push_back(file_info.data_file_name);
         }
 
-        writer->removeFilesAfterFailure(files_to_remove);
+        if (!checkLockFile(false))
+            return;
+
+        writer->removeFiles(files_to_remove);
+        removeLockFile();
     }
     catch (...)
     {

src/Backups/BackupImpl.h

@@ -47,9 +47,9 @@ public:
         const std::optional<BackupInfo> & base_backup_info_,
         std::shared_ptr<IBackupWriter> writer_,
         const ContextPtr & context_,
-        const std::optional<UUID> & backup_uuid_ = {},
         bool is_internal_backup_ = false,
-        const std::shared_ptr<IBackupCoordination> & coordination_ = {});
+        const std::shared_ptr<IBackupCoordination> & coordination_ = {},
+        const std::optional<UUID> & backup_uuid_ = {});
 
     ~BackupImpl() override;
 
@@ -76,12 +76,25 @@ private:
 
     void open(const ContextPtr & context);
     void close();
+
+    /// Writes the file ".backup" containing backup's metadata.
     void writeBackupMetadata();
     void readBackupMetadata();
+
+    /// Checks that a new backup doesn't exist yet.
+    void checkBackupDoesntExist() const;
+
+    /// Lock file named ".lock" and containing the UUID of a backup is used to own the place where we're writing the backup.
+    /// Thus it will not be allowed to put any other backup to the same place (even if the BACKUP command is executed on a different node).
+    void createLockFile();
+    bool checkLockFile(bool throw_if_failed) const;
+    void removeLockFile();
+
+    void removeAllFilesAfterFailure();
+
     String getArchiveNameWithSuffix(const String & suffix) const;
     std::shared_ptr<IArchiveReader> getArchiveReader(const String & suffix) const;
     std::shared_ptr<IArchiveWriter> getArchiveWriter(const String & suffix);
-    void removeAllFilesAfterFailure();
 
     const String backup_name;
     const ArchiveParams archive_params;
@@ -102,6 +115,8 @@ private:
     mutable std::unordered_map<String /* archive_suffix */, std::shared_ptr<IArchiveReader>> archive_readers;
     std::pair<String, std::shared_ptr<IArchiveWriter>> archive_writers[2];
     String current_archive_suffix;
+    String lock_file_name;
+    size_t num_files_written = 0;
     bool writing_finalized = false;
     const Poco::Logger * log;
 };

src/Backups/BackupSettings.cpp

@@ -5,6 +5,7 @@
 #include <Parsers/ASTSetQuery.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTLiteral.h>
+#include <IO/ReadHelpers.h>
 
 
 namespace DB
@@ -15,6 +16,48 @@ namespace ErrorCodes
     extern const int WRONG_BACKUP_SETTINGS;
 }
 
+
+namespace
+{
+    struct SettingFieldOptionalUUID
+    {
+        std::optional<UUID> value;
+
+        explicit SettingFieldOptionalUUID(const std::optional<UUID> & value_) : value(value_) {}
+
+        explicit SettingFieldOptionalUUID(const Field & field)
+        {
+            if (field.getType() == Field::Types::Null)
+            {
+                value = std::nullopt;
+                return;
+            }
+
+            if (field.getType() == Field::Types::String)
+            {
+                const String & str = field.get<const String &>();
+                if (str.empty())
+                {
+                    value = std::nullopt;
+                    return;
+                }
+
+                UUID id;
+                if (tryParse(id, str))
+                {
+                    value = id;
+                    return;
+                }
+            }
+
+            throw Exception(ErrorCodes::CANNOT_PARSE_BACKUP_SETTINGS, "Cannot parse uuid from {}", field);
+        }
+
+        explicit operator Field() const { return Field(value ? toString(*value) : ""); }
+    };
+}
+
+
 /// List of backup settings except base_backup_name and cluster_host_ids.
 #define LIST_OF_BACKUP_SETTINGS(M) \
     M(String, compression_method) \
@@ -26,7 +69,8 @@ namespace ErrorCodes
     M(UInt64, replica_num) \
     M(Bool, internal) \
     M(String, host_id) \
-    M(String, coordination_zk_path)
+    M(String, coordination_zk_path) \
+    M(OptionalUUID, backup_uuid)
 
 BackupSettings BackupSettings::fromBackupQuery(const ASTBackupQuery & query)
 {

src/Backups/BackupSettings.h

@@ -53,6 +53,10 @@ struct BackupSettings
     /// Path in Zookeeper used to coordinate a distributed backup created by BACKUP ON CLUSTER.
     String coordination_zk_path;
 
+    /// Internal, should not be specified by user.
+    /// UUID of the backup. If it's not set it will be generated randomly.
+    std::optional<UUID> backup_uuid;
+
     static BackupSettings fromBackupQuery(const ASTBackupQuery & query);
     void copySettingsToQuery(ASTBackupQuery & query) const;
 

src/Backups/BackupsWorker.cpp

@@ -72,12 +72,15 @@ UUID BackupsWorker::start(const ASTPtr & backup_or_restore_query, ContextMutable
 
 UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & context)
 {
-    UUID backup_uuid = UUIDHelpers::generateV4();
     auto backup_query = std::static_pointer_cast<ASTBackupQuery>(query->clone());
     auto backup_settings = BackupSettings::fromBackupQuery(*backup_query);
     auto backup_info = BackupInfo::fromAST(*backup_query->backup_name);
     bool on_cluster = !backup_query->cluster.empty();
 
+    if (!backup_settings.backup_uuid)
+        backup_settings.backup_uuid = UUIDHelpers::generateV4();
+    UUID backup_uuid = *backup_settings.backup_uuid;
+
     /// Prepare context to use.
     ContextPtr context_in_use = context;
     ContextMutablePtr mutable_context;
@@ -107,7 +110,7 @@ UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & c
         {
             if (async)
             {
-                query_scope.emplace(context_in_use);
+                query_scope.emplace(mutable_context);
                 setThreadName("BackupWorker");
             }
 
@@ -151,9 +154,9 @@ UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & c
             backup_create_params.compression_method = backup_settings.compression_method;
             backup_create_params.compression_level = backup_settings.compression_level;
             backup_create_params.password = backup_settings.password;
-            backup_create_params.backup_uuid = backup_uuid;
             backup_create_params.is_internal_backup = backup_settings.internal;
             backup_create_params.backup_coordination = backup_coordination;
+            backup_create_params.backup_uuid = backup_uuid;
             BackupMutablePtr backup = BackupFactory::instance().createBackup(backup_create_params);
 
             /// Write the backup.

src/Backups/registerBackupEnginesFileAndDisk.cpp

@@ -180,7 +180,7 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
                 writer = std::make_shared<BackupWriterFile>(path);
             else
                 writer = std::make_shared<BackupWriterDisk>(disk, path);
-            return std::make_unique<BackupImpl>(backup_name, archive_params, params.base_backup_info, writer, params.context, params.backup_uuid, params.is_internal_backup, params.backup_coordination);
+            return std::make_unique<BackupImpl>(backup_name, archive_params, params.base_backup_info, writer, params.context, params.is_internal_backup, params.backup_coordination, params.backup_uuid);
         }
     };
 

src/CMakeLists.txt

@@ -401,11 +401,11 @@ dbms_target_link_libraries (
         clickhouse_parsers
         ch_contrib::lz4
         Poco::JSON
-        Poco::MongoDB
         string_utils
     PUBLIC
         boost::system
         clickhouse_common_io
+        Poco::MongoDB
 )
 
 if (TARGET ch::mysqlxx)

src/Client/ClientBase.cpp

@@ -601,6 +601,7 @@ void ClientBase::initLogsOutputStream()
     {
         WriteBuffer * wb = out_logs_buf.get();
 
+        bool color_logs = false;
         if (!out_logs_buf)
         {
             if (server_logs_file.empty())
@@ -608,11 +609,13 @@ void ClientBase::initLogsOutputStream()
                 /// Use stderr by default
                 out_logs_buf = std::make_unique<WriteBufferFromFileDescriptor>(STDERR_FILENO);
                 wb = out_logs_buf.get();
+                color_logs = stderr_is_a_tty;
             }
             else if (server_logs_file == "-")
             {
                 /// Use stdout if --server_logs_file=- specified
                 wb = &std_out;
+                color_logs = stdout_is_a_tty;
             }
             else
             {
@@ -622,7 +625,7 @@ void ClientBase::initLogsOutputStream()
             }
         }
 
-        logs_out_stream = std::make_unique<InternalTextLogs>(*wb, stdout_is_a_tty);
+        logs_out_stream = std::make_unique<InternalTextLogs>(*wb, color_logs);
     }
 }
 
@@ -980,8 +983,7 @@ void ClientBase::onProfileEvents(Block & block)
             else if (event_name == MemoryTracker::USAGE_EVENT_NAME)
                 thread_times[host_name][thread_id].memory_usage = value;
         }
-        auto elapsed_time = profile_events.watch.elapsedMicroseconds();
-        progress_indication.updateThreadEventData(thread_times, elapsed_time);
+        progress_indication.updateThreadEventData(thread_times);
 
         if (need_render_progress)
             progress_indication.writeProgress();
@@ -2155,6 +2157,7 @@ void ClientBase::init(int argc, char ** argv)
 
     stdin_is_a_tty = isatty(STDIN_FILENO);
     stdout_is_a_tty = isatty(STDOUT_FILENO);
+    stderr_is_a_tty = isatty(STDERR_FILENO);
     terminal_width = getTerminalWidth();
 
     Arguments common_arguments{""}; /// 0th argument is ignored.

src/Client/ClientBase.h

@@ -173,6 +173,7 @@ protected:
 
     bool stdin_is_a_tty = false; /// stdin is a terminal.
     bool stdout_is_a_tty = false; /// stdout is a terminal.
+    bool stderr_is_a_tty = false; /// stderr is a terminal.
     uint64_t terminal_width = 0;
 
     ServerConnectionPtr connection;

src/Client/Suggest.cpp

@@ -50,52 +50,58 @@ static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggesti
 {
     /// NOTE: Once you will update the completion list,
     /// do not forget to update 01676_clickhouse_client_autocomplete.sh
-    WriteBufferFromOwnString query;
-    query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM ("
-        "SELECT name FROM system.functions"
-        " UNION ALL "
-        "SELECT name FROM system.table_engines"
-        " UNION ALL "
-        "SELECT name FROM system.formats"
-        " UNION ALL "
-        "SELECT name FROM system.table_functions"
-        " UNION ALL "
-        "SELECT name FROM system.data_type_families"
-        " UNION ALL "
-        "SELECT name FROM system.merge_tree_settings"
-        " UNION ALL "
-        "SELECT name FROM system.settings"
-        " UNION ALL ";
+    String query;
+
+    auto add_subquery = [&](std::string_view select, std::string_view result_column_name)
+    {
+        if (!query.empty())
+            query += " UNION ALL ";
+        query += fmt::format("SELECT * FROM viewIfPermitted({} ELSE null('{} String'))", select, result_column_name);
+    };
+
+    auto add_column = [&](std::string_view column_name, std::string_view table_name, bool distinct, std::optional<Int64> limit)
+    {
+        add_subquery(
+            fmt::format(
+                "SELECT {}{} FROM system.{}{}",
+                (distinct ? "DISTINCT " : ""),
+                column_name,
+                table_name,
+                (limit ? (" LIMIT " + std::to_string(*limit)) : "")),
+            column_name);
+    };
+
+    add_column("name", "functions", false, {});
+    add_column("name", "table_engines", false, {});
+    add_column("name", "formats", false, {});
+    add_column("name", "table_functions", false, {});
+    add_column("name", "data_type_families", false, {});
+    add_column("name", "merge_tree_settings", false, {});
+    add_column("name", "settings", false, {});
+
     if (!basic_suggestion)
     {
-        query << "SELECT cluster FROM system.clusters"
-                 " UNION ALL "
-                 "SELECT macro FROM system.macros"
-                 " UNION ALL "
-                 "SELECT policy_name FROM system.storage_policies"
-                 " UNION ALL ";
+        add_column("cluster", "clusters", false, {});
+        add_column("macro", "macros", false, {});
+        add_column("policy_name", "storage_policies", false, {});
     }
-    query << "SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate";
+
+    add_subquery("SELECT concat(func.name, comb.name) AS x FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate", "x");
+
     /// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero.
     if (suggestion_limit > 0)
     {
-        String limit_str = toString(suggestion_limit);
-        query << " UNION ALL "
-                 "SELECT name FROM system.databases LIMIT " << limit_str
-              << " UNION ALL "
-                 "SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
-              << " UNION ALL ";
-
+        add_column("name", "databases", false, suggestion_limit);
+        add_column("name", "tables", true, suggestion_limit);
         if (!basic_suggestion)
         {
-            query << "SELECT DISTINCT name FROM system.dictionaries LIMIT " << limit_str
-                  << " UNION ALL ";
+            add_column("name", "dictionaries", true, suggestion_limit);
         }
-        query << "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
+        add_column("name", "columns", true, suggestion_limit);
     }
-    query << ") WHERE notEmpty(res)";
 
-    return query.str();
+    query = "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + query + ") WHERE notEmpty(res)";
+    return query;
 }
 
 template <typename ConnectionType>

src/Columns/ColumnUnique.h

@@ -509,7 +509,7 @@ MutableColumnPtr ColumnUnique<ColumnType>::uniqueInsertRangeImpl(
     if (secondary_index)
         next_position += secondary_index->size();
 
-    auto insert_key = [&](const StringRef & ref, ReverseIndex<UInt64, ColumnType> & cur_index) -> MutableColumnPtr
+    auto insert_key = [&](StringRef ref, ReverseIndex<UInt64, ColumnType> & cur_index) -> MutableColumnPtr
     {
         auto inserted_pos = cur_index.insert(ref);
         positions[num_added_rows] = inserted_pos;

src/Columns/ReverseIndex.h

@@ -92,7 +92,7 @@ struct ReverseIndexHashTableCell
 
     /// Special case when we want to compare with something not in index_column.
     /// When we compare something inside column default keyEquals checks only that row numbers are equal.
-    bool keyEquals(const StringRef & object, size_t hash_ [[maybe_unused]], const State & state) const
+    bool keyEquals(StringRef object, size_t hash_ [[maybe_unused]], const State & state) const
     {
         auto index = key;
         if constexpr (has_base_index)
@@ -322,7 +322,7 @@ public:
     static constexpr bool is_numeric_column = isNumericColumn(static_cast<ColumnType *>(nullptr));
     static constexpr bool use_saved_hash = !is_numeric_column;
 
-    UInt64 insert(const StringRef & data);
+    UInt64 insert(StringRef data);
 
     /// Returns the found data's index in the dictionary. If index is not built, builds it.
     UInt64 getInsertionPoint(StringRef data)
@@ -383,7 +383,7 @@ private:
 
     void buildIndex();
 
-    UInt64 getHash(const StringRef & ref) const
+    UInt64 getHash(StringRef ref) const
     {
         if constexpr (is_numeric_column)
         {
@@ -478,7 +478,7 @@ ColumnUInt64::MutablePtr ReverseIndex<IndexType, ColumnType>::calcHashes() const
 }
 
 template <typename IndexType, typename ColumnType>
-UInt64 ReverseIndex<IndexType, ColumnType>::insert(const StringRef & data)
+UInt64 ReverseIndex<IndexType, ColumnType>::insert(StringRef data)
 {
     if (!index)
         buildIndex();

src/Common/CurrentThread.h

@@ -2,7 +2,6 @@
 
 #include <Interpreters/Context_fwd.h>
 #include <Common/ThreadStatus.h>
-#include <base/StringRef.h>
 
 #include <memory>
 #include <string>
@@ -76,7 +75,7 @@ public:
     static void finalizePerformanceCounters();
 
     /// Returns a non-empty string if the thread is attached to a query
-    static StringRef getQueryId()
+    static std::string_view getQueryId()
     {
         if (unlikely(!current_thread))
             return {};

src/Common/EventRateMeter.h

@@ -0,0 +1,63 @@
+#pragma once
+
+#include <base/defines.h>
+
+#include <Common/ExponentiallySmoothedCounter.h>
+
+#include <numbers>
+
+
+namespace DB
+{
+
+/// Event count measurement with exponential smoothing intended for computing time derivatives
+class EventRateMeter
+{
+public:
+    explicit EventRateMeter(double now, double period_)
+        : period(period_)
+        , half_decay_time(period * std::numbers::ln2) // for `ExponentiallySmoothedAverage::sumWeights()` to be equal to `1/period`
+    {
+        reset(now);
+    }
+
+    /// Add `count` events happened at `now` instant.
+    /// Previous events that are older than `period` from `now` will be forgotten
+    /// in a way to keep average event rate the same, using exponential smoothing.
+    /// NOTE: Adding events into distant past (further than `period`) must be avoided.
+    void add(double now, double count)
+    {
+        if (now - period <= start) // precise counting mode
+            events = ExponentiallySmoothedAverage(events.value + count, now);
+        else // exponential smoothing mode
+            events.add(count, now, half_decay_time);
+    }
+
+    /// Compute average event rate throughout `[now - period, now]` period.
+    /// If measurements are just started (`now - period < start`), then average
+    /// is computed based on shorter `[start; now]` period to avoid initial linear growth.
+    double rate(double now)
+    {
+        add(now, 0);
+        if (unlikely(now <= start))
+            return 0;
+        if (now - period <= start) // precise counting mode
+            return events.value / (now - start);
+        else // exponential smoothing mode
+            return events.get(half_decay_time); // equals to `events.value / period`
+    }
+
+    void reset(double now)
+    {
+        start = now;
+        events = ExponentiallySmoothedAverage();
+    }
+
+private:
+    const double period;
+    const double half_decay_time;
+    double start; // Instant in past without events before it; when measurement started or reset
+    ExponentiallySmoothedAverage events; // Estimated number of events in the last `period`
+};
+
+}

src/Common/FileSegment.cpp

@@ -104,10 +104,10 @@ String FileSegment::getCallerId()
 {
     if (!CurrentThread::isInitialized()
         || !CurrentThread::get().getQueryContext()
-        || CurrentThread::getQueryId().size == 0)
+        || CurrentThread::getQueryId().empty())
         return "None:" + toString(getThreadId());
 
-    return CurrentThread::getQueryId().toString() + ":" + toString(getThreadId());
+    return std::string(CurrentThread::getQueryId()) + ":" + toString(getThreadId());
 }
 
 String FileSegment::getOrSetDownloader()

src/Common/HashTable/StringHashMap.h

@@ -12,7 +12,7 @@ struct StringHashMapCell : public HashMapCell<Key, TMapped, StringHashTableHash,
     using Base::Base;
     static constexpr bool need_zero_value_storage = false;
     // external
-    const StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
+    StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
     // internal
     static const Key & getKey(const value_type & value_) { return value_.first; }
 };
@@ -32,7 +32,7 @@ struct StringHashMapCell<StringKey16, TMapped> : public HashMapCell<StringKey16,
     void setZero() { this->value.first.items[1] = 0; }
 
     // external
-    const StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
+    StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
     // internal
     static const StringKey16 & getKey(const value_type & value_) { return value_.first; }
 };
@@ -53,7 +53,7 @@ struct StringHashMapCell<StringKey24, TMapped> : public HashMapCell<StringKey24,
     void setZero() { this->value.first.c = 0; }
 
     // external
-    const StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
+    StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
     // internal
     static const StringKey24 & getKey(const value_type & value_) { return value_.first; }
 };

src/Common/IFileCache.cpp

@@ -58,7 +58,7 @@ static bool isQueryInitialized()
 {
     return CurrentThread::isInitialized()
         && CurrentThread::get().getQueryContext()
-        && CurrentThread::getQueryId().size != 0;
+        && !CurrentThread::getQueryId().empty();
 }
 
 bool IFileCache::isReadOnly()
@@ -77,7 +77,7 @@ IFileCache::QueryContextPtr IFileCache::getCurrentQueryContext(std::lock_guard<s
     if (!isQueryInitialized())
         return nullptr;
 
-    return getQueryContext(CurrentThread::getQueryId().toString(), cache_lock);
+    return getQueryContext(std::string(CurrentThread::getQueryId()), cache_lock);
 }
 
 IFileCache::QueryContextPtr IFileCache::getQueryContext(const String & query_id, std::lock_guard<std::mutex> & /* cache_lock */)

src/Common/JSONParsers/DummyJSONParser.h

@@ -84,7 +84,7 @@ struct DummyJSONParser
         static Iterator begin() { return {}; }
         static Iterator end() { return {}; }
         static size_t size() { return 0; }
-        bool find(const std::string_view &, Element &) const { return false; } /// NOLINT
+        bool find(std::string_view, Element &) const { return false; } /// NOLINT
 
 #if 0
         /// Optional: Provides access to an object's element by index.
@@ -93,7 +93,7 @@ struct DummyJSONParser
     };
 
     /// Parses a JSON document, returns the reference to its root element if succeeded.
-    bool parse(const std::string_view &, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
+    bool parse(std::string_view, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
 
 #if 0
     /// Optional: Allocates memory to parse JSON documents faster.

src/Common/JSONParsers/RapidJSONParser.h

@@ -98,7 +98,7 @@ struct RapidJSONParser
         ALWAYS_INLINE Iterator end() const { return ptr->MemberEnd(); }
         ALWAYS_INLINE size_t size() const { return ptr->MemberCount(); }
 
-        bool find(const std::string_view & key, Element & result) const
+        bool find(std::string_view key, Element & result) const
         {
             auto it = ptr->FindMember(rapidjson::StringRef(key.data(), key.length()));
             if (it == ptr->MemberEnd())
@@ -122,7 +122,7 @@ struct RapidJSONParser
     };
 
     /// Parses a JSON document, returns the reference to its root element if succeeded.
-    bool parse(const std::string_view & json, Element & result)
+    bool parse(std::string_view json, Element & result)
     {
         rapidjson::MemoryStream ms(json.data(), json.size());
         rapidjson::EncodedInputStream<rapidjson::UTF8<>, rapidjson::MemoryStream> is(ms);

src/Common/JSONParsers/SimdJSONParser.h

@@ -105,7 +105,7 @@ struct SimdJSONParser
         ALWAYS_INLINE Iterator end() const { return object.end(); }
         ALWAYS_INLINE size_t size() const { return object.size(); }
 
-        bool find(const std::string_view & key, Element & result) const
+        bool find(std::string_view key, Element & result) const
         {
             auto x = object.at_key(key);
             if (x.error())
@@ -131,7 +131,7 @@ struct SimdJSONParser
     };
 
     /// Parses a JSON document, returns the reference to its root element if succeeded.
-    bool parse(const std::string_view & json, Element & result)
+    bool parse(std::string_view json, Element & result)
     {
         auto document = parser.parse(json.data(), json.size());
         if (document.error())

src/Common/OpenSSLHelpers.cpp

@@ -10,7 +10,7 @@ namespace DB
 {
 #pragma GCC diagnostic warning "-Wold-style-cast"
 
-std::string encodeSHA256(const std::string_view & text)
+std::string encodeSHA256(std::string_view text)
 {
     return encodeSHA256(text.data(), text.size());
 }
@@ -21,7 +21,7 @@ std::string encodeSHA256(const void * text, size_t size)
     encodeSHA256(text, size, reinterpret_cast<unsigned char *>(out.data()));
     return out;
 }
-void encodeSHA256(const std::string_view & text, unsigned char * out)
+void encodeSHA256(std::string_view text, unsigned char * out)
 {
     encodeSHA256(text.data(), text.size(), out);
 }

src/Common/OpenSSLHelpers.h

@@ -10,10 +10,10 @@ namespace DB
 {
 
 /// Encodes `text` and returns it.
-std::string encodeSHA256(const std::string_view & text);
+std::string encodeSHA256(std::string_view text);
 std::string encodeSHA256(const void * text, size_t size);
 /// `out` must be at least 32 bytes long.
-void encodeSHA256(const std::string_view & text, unsigned char * out);
+void encodeSHA256(std::string_view text, unsigned char * out);
 void encodeSHA256(const void * text, size_t size, unsigned char * out);
 
 /// Returns concatenation of error strings for all errors that OpenSSL has recorded, emptying the error queue.

src/Common/ProgressIndication.cpp

@@ -8,6 +8,7 @@
 #include "Common/formatReadable.h"
 #include <Common/TerminalSize.h>
 #include <Common/UnicodeBar.h>
+#include <Common/Stopwatch.h>
 #include "IO/WriteBufferFromString.h"
 #include <Databases/DatabaseMemory.h>
 
@@ -16,16 +17,16 @@ namespace
 {
     constexpr UInt64 ALL_THREADS = 0;
 
-    double calculateCPUUsage(DB::ThreadIdToTimeMap times, UInt64 elapsed)
+    UInt64 aggregateCPUUsageNs(DB::ThreadIdToTimeMap times)
     {
-        auto accumulated = std::accumulate(times.begin(), times.end(), 0,
+        constexpr UInt64 us_to_ns = 1000;
+        return us_to_ns * std::accumulate(times.begin(), times.end(), 0ull,
         [](UInt64 acc, const auto & elem)
         {
             if (elem.first == ALL_THREADS)
                 return acc;
             return acc + elem.second.time();
         });
-        return static_cast<double>(accumulated) / elapsed;
     }
 }
 
@@ -55,7 +56,7 @@ void ProgressIndication::resetProgress()
     write_progress_on_update = false;
     {
         std::lock_guard lock(profile_events_mutex);
-        host_cpu_usage.clear();
+        cpu_usage_meter.reset(static_cast<double>(clock_gettime_ns()));
         thread_data.clear();
     }
 }
@@ -82,15 +83,17 @@ void ProgressIndication::addThreadIdToList(String const & host, UInt64 thread_id
     thread_to_times[thread_id] = {};
 }
 
-void ProgressIndication::updateThreadEventData(HostToThreadTimesMap & new_thread_data, UInt64 elapsed_time)
+void ProgressIndication::updateThreadEventData(HostToThreadTimesMap & new_thread_data)
 {
     std::lock_guard lock(profile_events_mutex);
 
+    UInt64 total_cpu_ns = 0;
     for (auto & new_host_map : new_thread_data)
     {
-        host_cpu_usage[new_host_map.first] = calculateCPUUsage(new_host_map.second, elapsed_time);
+        total_cpu_ns += aggregateCPUUsageNs(new_host_map.second);
         thread_data[new_host_map.first] = std::move(new_host_map.second);
     }
+    cpu_usage_meter.add(static_cast<double>(clock_gettime_ns()), total_cpu_ns);
 }
 
 size_t ProgressIndication::getUsedThreadsCount() const
@@ -104,14 +107,10 @@ size_t ProgressIndication::getUsedThreadsCount() const
         });
 }
 
-double ProgressIndication::getCPUUsage() const
+double ProgressIndication::getCPUUsage()
 {
     std::lock_guard lock(profile_events_mutex);
-
-    double res = 0;
-    for (const auto & elem : host_cpu_usage)
-        res += elem.second;
-    return res;
+    return cpu_usage_meter.rate(clock_gettime_ns());
 }
 
 ProgressIndication::MemoryUsage ProgressIndication::getMemoryUsage() const

src/Common/ProgressIndication.h

@@ -7,7 +7,7 @@
 #include <Interpreters/Context.h>
 #include <base/types.h>
 #include <Common/Stopwatch.h>
-
+#include <Common/EventRateMeter.h>
 
 /// http://en.wikipedia.org/wiki/ANSI_escape_code
 #define CLEAR_TO_END_OF_LINE "\033[K"
@@ -59,12 +59,12 @@ public:
 
     void addThreadIdToList(String const & host, UInt64 thread_id);
 
-    void updateThreadEventData(HostToThreadTimesMap & new_thread_data, UInt64 elapsed_time);
+    void updateThreadEventData(HostToThreadTimesMap & new_thread_data);
 
 private:
     size_t getUsedThreadsCount() const;
 
-    double getCPUUsage() const;
+    double getCPUUsage();
 
     struct MemoryUsage
     {
@@ -91,7 +91,7 @@ private:
 
     bool write_progress_on_update = false;
 
-    std::unordered_map<String, double> host_cpu_usage;
+    EventRateMeter cpu_usage_meter{static_cast<double>(clock_gettime_ns()), 3'000'000'000 /*ns*/}; // average cpu utilization last 3 second
     HostToThreadTimesMap thread_data;
     /// In case of all of the above:
     /// - clickhouse-local
@@ -100,7 +100,7 @@ private:
     ///
     /// It is possible concurrent access to the following:
     /// - writeProgress() (class properties) (guarded with progress_mutex)
-    /// - thread_data/host_cpu_usage (guarded with profile_events_mutex)
+    /// - thread_data/cpu_usage_meter (guarded with profile_events_mutex)
     mutable std::mutex profile_events_mutex;
     mutable std::mutex progress_mutex;
 };

src/Common/SettingsChanges.cpp

@@ -4,7 +4,7 @@ namespace DB
 {
 namespace
 {
-    SettingChange * find(SettingsChanges & changes, const std::string_view & name)
+    SettingChange * find(SettingsChanges & changes, std::string_view name)
     {
         auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
         if (it == changes.end())
@@ -12,7 +12,7 @@ namespace
         return &*it;
     }
 
-    const SettingChange * find(const SettingsChanges & changes, const std::string_view & name)
+    const SettingChange * find(const SettingsChanges & changes, std::string_view name)
     {
         auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
         if (it == changes.end())
@@ -21,7 +21,7 @@ namespace
     }
 }
 
-bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) const
+bool SettingsChanges::tryGet(std::string_view name, Field & out_value) const
 {
     const auto * change = find(*this, name);
     if (!change)
@@ -30,7 +30,7 @@ bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) c
     return true;
 }
 
-const Field * SettingsChanges::tryGet(const std::string_view & name) const
+const Field * SettingsChanges::tryGet(std::string_view name) const
 {
     const auto * change = find(*this, name);
     if (!change)
@@ -38,7 +38,7 @@ const Field * SettingsChanges::tryGet(const std::string_view & name) const
     return &change->value;
 }
 
-Field * SettingsChanges::tryGet(const std::string_view & name)
+Field * SettingsChanges::tryGet(std::string_view name)
 {
     auto * change = find(*this, name);
     if (!change)

src/Common/SettingsChanges.h

@@ -14,8 +14,8 @@ struct SettingChange
     Field value;
 
     SettingChange() = default;
-    SettingChange(const std::string_view & name_, const Field & value_) : name(name_), value(value_) {}
-    SettingChange(const std::string_view & name_, Field && value_) : name(name_), value(std::move(value_)) {}
+    SettingChange(std::string_view name_, const Field & value_) : name(name_), value(value_) {}
+    SettingChange(std::string_view name_, Field && value_) : name(name_), value(std::move(value_)) {}
 
     friend bool operator ==(const SettingChange & lhs, const SettingChange & rhs) { return (lhs.name == rhs.name) && (lhs.value == rhs.value); }
     friend bool operator !=(const SettingChange & lhs, const SettingChange & rhs) { return !(lhs == rhs); }
@@ -27,9 +27,9 @@ class SettingsChanges : public std::vector<SettingChange>
 public:
     using std::vector<SettingChange>::vector;
 
-    bool tryGet(const std::string_view & name, Field & out_value) const;
-    const Field * tryGet(const std::string_view & name) const;
-    Field * tryGet(const std::string_view & name);
+    bool tryGet(std::string_view name, Field & out_value) const;
+    const Field * tryGet(std::string_view name) const;
+    Field * tryGet(std::string_view name);
 };
 
 }

src/Common/ShellCommand.cpp

@@ -1,9 +1,7 @@
 #include <sys/types.h>
 #include <sys/wait.h>
-#include <fcntl.h>
 #include <dlfcn.h>
 #include <unistd.h>
-#include <ctime>
 #include <csignal>
 
 #include <Common/logger_useful.h>
@@ -13,6 +11,7 @@
 #include <Common/PipeFDs.h>
 #include <IO/WriteHelpers.h>
 #include <IO/Operators.h>
+#include <Common/waitForPid.h>
 
 
 namespace
@@ -94,53 +93,15 @@ ShellCommand::~ShellCommand()
 
 bool ShellCommand::tryWaitProcessWithTimeout(size_t timeout_in_seconds)
 {
-    int status = 0;
-
     LOG_TRACE(getLogger(), "Try wait for shell command pid {} with timeout {}", pid, timeout_in_seconds);
 
     wait_called = true;
-    struct timespec interval {.tv_sec = 1, .tv_nsec = 0};
 
     in.close();
     out.close();
     err.close();
 
-    if (timeout_in_seconds == 0)
-    {
-        /// If there is no timeout before signal try to waitpid 1 time without block so we can avoid sending
-        /// signal if process is already normally terminated.
-
-        int waitpid_res = waitpid(pid, &status, WNOHANG);
-        bool process_terminated_normally = (waitpid_res == pid);
-        return process_terminated_normally;
-    }
-
-    /// If timeout is positive try waitpid without block in loop until
-    /// process is normally terminated or waitpid return error
-
-    while (timeout_in_seconds != 0)
-    {
-        int waitpid_res = waitpid(pid, &status, WNOHANG);
-        bool process_terminated_normally = (waitpid_res == pid);
-
-        if (process_terminated_normally)
-        {
-            return true;
-        }
-        else if (waitpid_res == 0)
-        {
-            --timeout_in_seconds;
-            nanosleep(&interval, nullptr);
-
-            continue;
-        }
-        else if (waitpid_res == -1 && errno != EINTR)
-        {
-            return false;
-        }
-    }
-
-    return false;
+    return waitForPid(pid, timeout_in_seconds);
 }
 
 void ShellCommand::logCommand(const char * filename, char * const argv[])

src/Common/SpaceSaving.h

@@ -49,12 +49,12 @@ struct SpaceSavingArena
 template <>
 struct SpaceSavingArena<StringRef>
 {
-    StringRef emplace(const StringRef & key)
+    StringRef emplace(StringRef key)
     {
         return copyStringInArena(arena, key);
     }
 
-    void free(const StringRef & key)
+    void free(StringRef key)
     {
         if (key.data)
             arena.free(const_cast<char *>(key.data), key.size);

src/Common/StringUtils/StringUtils.h

@@ -147,7 +147,7 @@ inline bool isPunctuationASCII(char c)
 }
 
 
-inline bool isValidIdentifier(const std::string_view & str)
+inline bool isValidIdentifier(std::string_view str)
 {
     return !str.empty()
         && isValidIdentifierBegin(str[0])

src/Common/TLDListsHolder.cpp

@@ -20,13 +20,13 @@ TLDList::TLDList(size_t size)
     : tld_container(size)
     , pool(std::make_unique<Arena>(10 << 20))
 {}
-bool TLDList::insert(const StringRef & host)
+bool TLDList::insert(StringRef host)
 {
     bool inserted;
     tld_container.emplace(DB::ArenaKeyHolder{host, *pool}, inserted);
     return inserted;
 }
-bool TLDList::has(const StringRef & host) const
+bool TLDList::has(StringRef host) const
 {
     return tld_container.has(host);
 }

src/Common/TLDListsHolder.h

@@ -23,9 +23,9 @@ public:
     explicit TLDList(size_t size);
 
     /// Return true if the tld_container does not contains such element.
-    bool insert(const StringRef & host);
+    bool insert(StringRef host);
     /// Check is there such TLD
-    bool has(const StringRef & host) const;
+    bool has(StringRef host) const;
     size_t size() const { return tld_container.size(); }
 
 private:

src/Common/TargetSpecific.cpp

@@ -18,6 +18,8 @@ UInt32 getSupportedArchs()
         result |= static_cast<UInt32>(TargetArch::AVX512F);
     if (Cpu::CpuFlagsCache::have_AVX512BW)
         result |= static_cast<UInt32>(TargetArch::AVX512BW);
+    if (Cpu::CpuFlagsCache::have_AVX512VBMI)
+        result |= static_cast<UInt32>(TargetArch::AVX512VBMI);
     return result;
 }
 
@@ -37,6 +39,7 @@ String toString(TargetArch arch)
         case TargetArch::AVX2:    return "avx2";
         case TargetArch::AVX512F: return "avx512f";
         case TargetArch::AVX512BW: return "avx512bw";
+        case TargetArch::AVX512VBMI: return "avx512vbmi";
     }
 
     __builtin_unreachable();

src/Common/TargetSpecific.h

@@ -81,6 +81,7 @@ enum class TargetArch : UInt32
     AVX2     = (1 << 2),
     AVX512F  = (1 << 3),
     AVX512BW  = (1 << 4),
+    AVX512VBMI = (1 << 5),
 };
 
 /// Runtime detection.
@@ -88,6 +89,10 @@ bool isArchSupported(TargetArch arch);
 
 String toString(TargetArch arch);
 
+#ifndef ENABLE_MULTITARGET_CODE
+#   define ENABLE_MULTITARGET_CODE 0
+#endif
+
 #if ENABLE_MULTITARGET_CODE && defined(__GNUC__) && defined(__x86_64__)
 
 /// NOLINTNEXTLINE
@@ -95,6 +100,7 @@ String toString(TargetArch arch);
 
 #if defined(__clang__)
 
+#define AVX512VBMI_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi")))
 #define AVX512BW_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw")))
 #define AVX512_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f")))
 #define AVX2_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2")))
@@ -102,6 +108,8 @@ String toString(TargetArch arch);
 #define SSE42_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt")))
 #define DEFAULT_FUNCTION_SPECIFIC_ATTRIBUTE
 
+#   define BEGIN_AVX512VBMI_SPECIFIC_CODE \
+        _Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi\"))),apply_to=function)")
 #   define BEGIN_AVX512BW_SPECIFIC_CODE \
         _Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw\"))),apply_to=function)")
 #   define BEGIN_AVX512F_SPECIFIC_CODE \
@@ -121,13 +129,17 @@ String toString(TargetArch arch);
 #   define DUMMY_FUNCTION_DEFINITION [[maybe_unused]] void _dummy_function_definition();
 #else
 
+#define AVX512VBMI_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,tune=native")))
 #define AVX512BW_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,tune=native")))
 #define AVX512_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,tune=native")))
 #define AVX2_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,tune=native")))
 #define AVX_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,tune=native")))
-#define SSE42_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt",tune=native))))
+#define SSE42_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt",tune=native)))
 #define DEFAULT_FUNCTION_SPECIFIC_ATTRIBUTE
 
+#   define BEGIN_AVX512VBMI_SPECIFIC_CODE \
+        _Pragma("GCC push_options") \
+        _Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,tune=native\")")
 #   define BEGIN_AVX512BW_SPECIFIC_CODE \
         _Pragma("GCC push_options") \
         _Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,tune=native\")")
@@ -196,6 +208,15 @@ namespace TargetSpecific::AVX512BW { \
 } \
 END_TARGET_SPECIFIC_CODE
 
+#define DECLARE_AVX512VBMI_SPECIFIC_CODE(...) \
+BEGIN_AVX512VBMI_SPECIFIC_CODE \
+namespace TargetSpecific::AVX512VBMI { \
+    DUMMY_FUNCTION_DEFINITION \
+    using namespace DB::TargetSpecific::AVX512VBMI; \
+    __VA_ARGS__ \
+} \
+END_TARGET_SPECIFIC_CODE
+
 #else
 
 #define USE_MULTITARGET_CODE 0
@@ -207,6 +228,7 @@ END_TARGET_SPECIFIC_CODE
 #define DECLARE_AVX2_SPECIFIC_CODE(...)
 #define DECLARE_AVX512F_SPECIFIC_CODE(...)
 #define DECLARE_AVX512BW_SPECIFIC_CODE(...)
+#define DECLARE_AVX512VBMI_SPECIFIC_CODE(...)
 
 #endif
 
@@ -223,7 +245,8 @@ DECLARE_SSE42_SPECIFIC_CODE  (__VA_ARGS__) \
 DECLARE_AVX_SPECIFIC_CODE    (__VA_ARGS__) \
 DECLARE_AVX2_SPECIFIC_CODE   (__VA_ARGS__) \
 DECLARE_AVX512F_SPECIFIC_CODE(__VA_ARGS__) \
-DECLARE_AVX512BW_SPECIFIC_CODE(__VA_ARGS__)
+DECLARE_AVX512BW_SPECIFIC_CODE(__VA_ARGS__) \
+DECLARE_AVX512VBMI_SPECIFIC_CODE(__VA_ARGS__)
 
 DECLARE_DEFAULT_CODE(
     constexpr auto BuildArch = TargetArch::Default; /// NOLINT
@@ -249,6 +272,11 @@ DECLARE_AVX512BW_SPECIFIC_CODE(
     constexpr auto BuildArch = TargetArch::AVX512BW; /// NOLINT
 ) // DECLARE_AVX512BW_SPECIFIC_CODE
 
+DECLARE_AVX512VBMI_SPECIFIC_CODE(
+    constexpr auto BuildArch = TargetArch::AVX512VBMI; /// NOLINT
+) // DECLARE_AVX512VBMI_SPECIFIC_CODE
+
+
 /** Runtime Dispatch helpers for class members.
   *
   * Example of usage:

src/Common/TaskStatsInfoGetter.cpp

@@ -21,6 +21,7 @@
 
 #if defined(__clang__)
     #pragma clang diagnostic ignored "-Wgnu-anonymous-struct"
+    #pragma clang diagnostic ignored "-Wnested-anon-types"
 #endif
 
 /// Basic idea is motivated by "iotop" tool.

src/Common/ThreadPool.h

@@ -162,20 +162,19 @@ public:
 
     template <typename Function, typename... Args>
     explicit ThreadFromGlobalPool(Function && func, Args &&... args)
-        : state(std::make_shared<Poco::Event>())
-        , thread_id(std::make_shared<std::thread::id>())
+        : state(std::make_shared<State>())
     {
-        /// NOTE: If this will throw an exception, the destructor won't be called.
+        /// NOTE:
+        /// - If this will throw an exception, the destructor won't be called
+        /// - this pointer cannot be passed in the lambda, since after detach() it will not be valid
         GlobalThreadPool::instance().scheduleOrThrow([
-            thread_id = thread_id,
             state = state,
             func = std::forward<Function>(func),
             args = std::make_tuple(std::forward<Args>(args)...)]() mutable /// mutable is needed to destroy capture
         {
-            auto event = std::move(state);
-            SCOPE_EXIT(event->set());
+            SCOPE_EXIT(state->event.set());
 
-            thread_id = std::make_shared<std::thread::id>(std::this_thread::get_id());
+            state->thread_id = std::this_thread::get_id();
 
             /// This moves are needed to destroy function and arguments before exit.
             /// It will guarantee that after ThreadFromGlobalPool::join all captured params are destroyed.
@@ -196,31 +195,30 @@ public:
 
     ThreadFromGlobalPool & operator=(ThreadFromGlobalPool && rhs) noexcept
     {
-        if (joinable())
+        if (initialized())
             abort();
         state = std::move(rhs.state);
-        thread_id = std::move(rhs.thread_id);
         return *this;
     }
 
     ~ThreadFromGlobalPool()
     {
-        if (joinable())
+        if (initialized())
             abort();
     }
 
     void join()
     {
-        if (!joinable())
+        if (!initialized())
             abort();
 
-        state->wait();
+        state->event.wait();
         state.reset();
     }
 
     void detach()
     {
-        if (!joinable())
+        if (!initialized())
             abort();
         state.reset();
     }
@@ -230,15 +228,30 @@ public:
         if (!state)
             return false;
         /// Thread cannot join itself.
-        if (*thread_id == std::this_thread::get_id())
+        if (state->thread_id == std::this_thread::get_id())
             return false;
         return true;
     }
 
 private:
-    /// The state used in this object and inside the thread job.
-    std::shared_ptr<Poco::Event> state;
-    std::shared_ptr<std::thread::id> thread_id;
+    struct State
+    {
+        /// Should be atomic() because of possible concurrent access between
+        /// assignment and joinable() check.
+        std::atomic<std::thread::id> thread_id;
+
+        /// The state used in this object and inside the thread job.
+        Poco::Event event;
+    };
+    std::shared_ptr<State> state;
+
+    /// Internally initialized() should be used over joinable(),
+    /// since it is enough to know that the thread is initialized,
+    /// and ignore that fact that thread cannot join itself.
+    bool initialized() const
+    {
+        return static_cast<bool>(state);
+    }
 };
 
 

src/Common/ThreadStatus.h

@@ -210,7 +210,7 @@ public:
         return thread_state.load(std::memory_order_relaxed);
     }
 
-    StringRef getQueryId() const
+    std::string_view getQueryId() const
     {
         return query_id;
     }

src/Common/Throttler.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <Common/Throttler_fwd.h>
+
 #include <mutex>
 #include <memory>
 #include <base/sleep.h>
@@ -57,7 +59,4 @@ private:
     std::shared_ptr<Throttler> parent;
 };
 
-
-using ThrottlerPtr = std::shared_ptr<Throttler>;
-
 }

src/Common/Throttler_fwd.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <memory>
+
+namespace DB
+{
+
+class Throttler;
+using ThrottlerPtr = std::shared_ptr<Throttler>;
+
+}