Merge branch 'master' into fix-50669

.github/workflows/master.yml

@@ -850,6 +850,48 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderBinRISCV64:
+    needs: [DockerHubPush]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          BUILD_NAME=binary_riscv64
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v3
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
 ##################################### Docker images  #######################################
 ############################################################################################
@@ -932,6 +974,7 @@ jobs:
       - BuilderBinDarwinAarch64
       - BuilderBinFreeBSD
       - BuilderBinPPC64
+      - BuilderBinRISCV64
       - BuilderBinAmd64Compat
       - BuilderBinAarch64V80Compat
       - BuilderBinClangTidy

.github/workflows/pull_request.yml

@@ -911,6 +911,47 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  BuilderBinRISCV64:
+    needs: [DockerHubPush, FastTest, StyleCheck]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          BUILD_NAME=binary_riscv64
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v3
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+          submodules: true
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+      - name: Upload build URLs to artifacts
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.BUILD_URLS }}
+          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
 ##################################### Docker images  #######################################
 ############################################################################################
@@ -992,6 +1033,7 @@ jobs:
       - BuilderBinDarwinAarch64
       - BuilderBinFreeBSD
       - BuilderBinPPC64
+      - BuilderBinRISCV64
       - BuilderBinAmd64Compat
       - BuilderBinAarch64V80Compat
       - BuilderBinClangTidy

README.md

@@ -16,8 +16,9 @@ curl https://clickhouse.com/ | sh
 * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
 * [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
 * [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events.
-* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation.
-* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev.
+* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlighting and navigation.
+* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev.
+* [Static Analysis (SonarCloud)](https://sonarcloud.io/project/issues?resolved=false&id=ClickHouse_ClickHouse) proposes C++ quality improvements.
 * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
 
 ## Upcoming Events

base/base/hex.h

@@ -4,212 +4,288 @@
 #include <cstring>
 #include "types.h"
 
-/// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly.
+namespace CityHash_v1_0_2 { struct uint128; }
 
-constexpr inline std::string_view hex_digit_to_char_uppercase_table = "0123456789ABCDEF";
-constexpr inline std::string_view hex_digit_to_char_lowercase_table = "0123456789abcdef";
+namespace wide
+{
+    template <size_t Bits, typename Signed>
+    class integer;
+}
+
+namespace impl
+{
+    /// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly.
+    constexpr inline std::string_view hex_digit_to_char_uppercase_table = "0123456789ABCDEF";
+    constexpr inline std::string_view hex_digit_to_char_lowercase_table = "0123456789abcdef";
+
+    /// Maps 0..255 to 00..FF or 00..ff correspondingly.
+    constexpr inline std::string_view hex_byte_to_char_uppercase_table = //
+        "000102030405060708090A0B0C0D0E0F"
+        "101112131415161718191A1B1C1D1E1F"
+        "202122232425262728292A2B2C2D2E2F"
+        "303132333435363738393A3B3C3D3E3F"
+        "404142434445464748494A4B4C4D4E4F"
+        "505152535455565758595A5B5C5D5E5F"
+        "606162636465666768696A6B6C6D6E6F"
+        "707172737475767778797A7B7C7D7E7F"
+        "808182838485868788898A8B8C8D8E8F"
+        "909192939495969798999A9B9C9D9E9F"
+        "A0A1A2A3A4A5A6A7A8A9AAABACADAEAF"
+        "B0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF"
+        "C0C1C2C3C4C5C6C7C8C9CACBCCCDCECF"
+        "D0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF"
+        "E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEF"
+        "F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF";
+
+    constexpr inline std::string_view hex_byte_to_char_lowercase_table = //
+        "000102030405060708090a0b0c0d0e0f"
+        "101112131415161718191a1b1c1d1e1f"
+        "202122232425262728292a2b2c2d2e2f"
+        "303132333435363738393a3b3c3d3e3f"
+        "404142434445464748494a4b4c4d4e4f"
+        "505152535455565758595a5b5c5d5e5f"
+        "606162636465666768696a6b6c6d6e6f"
+        "707172737475767778797a7b7c7d7e7f"
+        "808182838485868788898a8b8c8d8e8f"
+        "909192939495969798999a9b9c9d9e9f"
+        "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
+        "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"
+        "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf"
+        "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf"
+        "e0e1e2e3e4e5e6e7e8e9eaebecedeeef"
+        "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff";
+
+    /// Maps 0..255 to 00000000..11111111 correspondingly.
+    constexpr inline std::string_view bin_byte_to_char_table = //
+        "0000000000000001000000100000001100000100000001010000011000000111"
+        "0000100000001001000010100000101100001100000011010000111000001111"
+        "0001000000010001000100100001001100010100000101010001011000010111"
+        "0001100000011001000110100001101100011100000111010001111000011111"
+        "0010000000100001001000100010001100100100001001010010011000100111"
+        "0010100000101001001010100010101100101100001011010010111000101111"
+        "0011000000110001001100100011001100110100001101010011011000110111"
+        "0011100000111001001110100011101100111100001111010011111000111111"
+        "0100000001000001010000100100001101000100010001010100011001000111"
+        "0100100001001001010010100100101101001100010011010100111001001111"
+        "0101000001010001010100100101001101010100010101010101011001010111"
+        "0101100001011001010110100101101101011100010111010101111001011111"
+        "0110000001100001011000100110001101100100011001010110011001100111"
+        "0110100001101001011010100110101101101100011011010110111001101111"
+        "0111000001110001011100100111001101110100011101010111011001110111"
+        "0111100001111001011110100111101101111100011111010111111001111111"
+        "1000000010000001100000101000001110000100100001011000011010000111"
+        "1000100010001001100010101000101110001100100011011000111010001111"
+        "1001000010010001100100101001001110010100100101011001011010010111"
+        "1001100010011001100110101001101110011100100111011001111010011111"
+        "1010000010100001101000101010001110100100101001011010011010100111"
+        "1010100010101001101010101010101110101100101011011010111010101111"
+        "1011000010110001101100101011001110110100101101011011011010110111"
+        "1011100010111001101110101011101110111100101111011011111010111111"
+        "1100000011000001110000101100001111000100110001011100011011000111"
+        "1100100011001001110010101100101111001100110011011100111011001111"
+        "1101000011010001110100101101001111010100110101011101011011010111"
+        "1101100011011001110110101101101111011100110111011101111011011111"
+        "1110000011100001111000101110001111100100111001011110011011100111"
+        "1110100011101001111010101110101111101100111011011110111011101111"
+        "1111000011110001111100101111001111110100111101011111011011110111"
+        "1111100011111001111110101111101111111100111111011111111011111111";
+
+    /// Maps 0..9, A..F, a..f to 0..15. Other chars are mapped to implementation specific value.
+    constexpr inline std::string_view hex_char_to_digit_table
+        = {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" //0-9
+        "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //A-Z
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //a-z
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff",
+        256};
+
+    /// Converts a hex digit '0'..'f' or '0'..'F' to its value 0..15.
+    constexpr UInt8 unhexDigit(char c)
+    {
+        return hex_char_to_digit_table[static_cast<UInt8>(c)];
+    }
+
+    /// Converts an unsigned integer in the native endian to hexadecimal representation and back. Used as a base class for HexConversion<T>.
+    template <typename TUInt, typename = void>
+    struct HexConversionUInt
+    {
+        static const constexpr size_t num_hex_digits = sizeof(TUInt) * 2;
+
+        static void hex(TUInt uint_, char * out, std::string_view table)
+        {
+            union
+            {
+                TUInt value;
+                UInt8 uint8[sizeof(TUInt)];
+            };
+
+            value = uint_;
+
+            for (size_t i = 0; i < sizeof(TUInt); ++i)
+            {
+                if constexpr (std::endian::native == std::endian::little)
+                    memcpy(out + i * 2, &table[static_cast<size_t>(uint8[sizeof(TUInt) - 1 - i]) * 2], 2);
+                else
+                    memcpy(out + i * 2, &table[static_cast<size_t>(uint8[i]) * 2], 2);
+            }
+        }
+
+        static TUInt unhex(const char * data)
+        {
+            TUInt res;
+            if constexpr (sizeof(TUInt) == 1)
+            {
+                res = static_cast<UInt8>(unhexDigit(data[0])) * 0x10 + static_cast<UInt8>(unhexDigit(data[1]));
+            }
+            else if constexpr (sizeof(TUInt) == 2)
+            {
+                res = static_cast<UInt16>(unhexDigit(data[0])) * 0x1000 + static_cast<UInt16>(unhexDigit(data[1])) * 0x100
+                    + static_cast<UInt16>(unhexDigit(data[2])) * 0x10 + static_cast<UInt16>(unhexDigit(data[3]));
+            }
+            else if constexpr ((sizeof(TUInt) <= 8) || ((sizeof(TUInt) % 8) != 0))
+            {
+                res = 0;
+                for (size_t i = 0; i < sizeof(TUInt) * 2; ++i, ++data)
+                {
+                    res <<= 4;
+                    res += unhexDigit(*data);
+                }
+            }
+            else
+            {
+                res = 0;
+                for (size_t i = 0; i < sizeof(TUInt) / 8; ++i, data += 16)
+                {
+                    res <<= 64;
+                    res += HexConversionUInt<UInt64>::unhex(data);
+                }
+            }
+            return res;
+        }
+    };
+
+    /// Helper template class to convert a value of any supported type to hexadecimal representation and back.
+    template <typename T, typename SFINAE = void>
+    struct HexConversion;
+
+    template <typename TUInt>
+    struct HexConversion<TUInt, std::enable_if_t<std::is_integral_v<TUInt>>> : public HexConversionUInt<TUInt> {};
+
+    template <size_t Bits, typename Signed>
+    struct HexConversion<wide::integer<Bits, Signed>> : public HexConversionUInt<wide::integer<Bits, Signed>> {};
+
+    template <typename CityHashUInt128> /// Partial specialization here allows not to include <city.h> in this header.
+    struct HexConversion<CityHashUInt128, std::enable_if_t<std::is_same_v<CityHashUInt128, typename CityHash_v1_0_2::uint128>>>
+    {
+        static const constexpr size_t num_hex_digits = 32;
+
+        static void hex(const CityHashUInt128 & uint_, char * out, std::string_view table)
+        {
+            HexConversion<UInt64>::hex(uint_.high64, out, table);
+            HexConversion<UInt64>::hex(uint_.low64, out + 16, table);
+        }
+
+        static CityHashUInt128 unhex(const char * data)
+        {
+            CityHashUInt128 res;
+            res.high64 = HexConversion<UInt64>::unhex(data);
+            res.low64 = HexConversion<UInt64>::unhex(data + 16);
+            return res;
+        }
+    };
+}
+
+/// Produces a hexadecimal representation of an integer value with leading zeros (for checksums).
+/// The function supports native integer types, wide::integer, CityHash_v1_0_2::uint128.
+/// It can be used with signed types as well, however they are written as corresponding unsigned numbers
+/// using two's complement (i.e. for example "-1" is written as "0xFF", not as "-0x01").
+template <typename T>
+void writeHexUIntUppercase(const T & value, char * out)
+{
+    impl::HexConversion<T>::hex(value, out, impl::hex_byte_to_char_uppercase_table);
+}
+
+template <typename T>
+void writeHexUIntLowercase(const T & value, char * out)
+{
+    impl::HexConversion<T>::hex(value, out, impl::hex_byte_to_char_lowercase_table);
+}
+
+template <typename T>
+std::string getHexUIntUppercase(const T & value)
+{
+    std::string res(impl::HexConversion<T>::num_hex_digits, '\0');
+    writeHexUIntUppercase(value, res.data());
+    return res;
+}
+
+template <typename T>
+std::string getHexUIntLowercase(const T & value)
+{
+    std::string res(impl::HexConversion<T>::num_hex_digits, '\0');
+    writeHexUIntLowercase(value, res.data());
+    return res;
+}
 
 constexpr char hexDigitUppercase(unsigned char c)
 {
-    return hex_digit_to_char_uppercase_table[c];
+    return impl::hex_digit_to_char_uppercase_table[c];
 }
+
 constexpr char hexDigitLowercase(unsigned char c)
 {
-    return hex_digit_to_char_lowercase_table[c];
+    return impl::hex_digit_to_char_lowercase_table[c];
 }
 
-/// Maps 0..255 to 00..FF or 00..ff correspondingly
-
-constexpr inline std::string_view hex_byte_to_char_uppercase_table = //
-    "000102030405060708090A0B0C0D0E0F"
-    "101112131415161718191A1B1C1D1E1F"
-    "202122232425262728292A2B2C2D2E2F"
-    "303132333435363738393A3B3C3D3E3F"
-    "404142434445464748494A4B4C4D4E4F"
-    "505152535455565758595A5B5C5D5E5F"
-    "606162636465666768696A6B6C6D6E6F"
-    "707172737475767778797A7B7C7D7E7F"
-    "808182838485868788898A8B8C8D8E8F"
-    "909192939495969798999A9B9C9D9E9F"
-    "A0A1A2A3A4A5A6A7A8A9AAABACADAEAF"
-    "B0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF"
-    "C0C1C2C3C4C5C6C7C8C9CACBCCCDCECF"
-    "D0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF"
-    "E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEF"
-    "F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF";
-
-constexpr inline std::string_view hex_byte_to_char_lowercase_table = //
-    "000102030405060708090a0b0c0d0e0f"
-    "101112131415161718191a1b1c1d1e1f"
-    "202122232425262728292a2b2c2d2e2f"
-    "303132333435363738393a3b3c3d3e3f"
-    "404142434445464748494a4b4c4d4e4f"
-    "505152535455565758595a5b5c5d5e5f"
-    "606162636465666768696a6b6c6d6e6f"
-    "707172737475767778797a7b7c7d7e7f"
-    "808182838485868788898a8b8c8d8e8f"
-    "909192939495969798999a9b9c9d9e9f"
-    "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
-    "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"
-    "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf"
-    "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf"
-    "e0e1e2e3e4e5e6e7e8e9eaebecedeeef"
-    "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff";
-
 inline void writeHexByteUppercase(UInt8 byte, void * out)
 {
-    memcpy(out, &hex_byte_to_char_uppercase_table[static_cast<size_t>(byte) * 2], 2);
+    memcpy(out, &impl::hex_byte_to_char_uppercase_table[static_cast<size_t>(byte) * 2], 2);
 }
 
 inline void writeHexByteLowercase(UInt8 byte, void * out)
 {
-    memcpy(out, &hex_byte_to_char_lowercase_table[static_cast<size_t>(byte) * 2], 2);
+    memcpy(out, &impl::hex_byte_to_char_lowercase_table[static_cast<size_t>(byte) * 2], 2);
 }
 
-constexpr inline std::string_view bin_byte_to_char_table = //
-    "0000000000000001000000100000001100000100000001010000011000000111"
-    "0000100000001001000010100000101100001100000011010000111000001111"
-    "0001000000010001000100100001001100010100000101010001011000010111"
-    "0001100000011001000110100001101100011100000111010001111000011111"
-    "0010000000100001001000100010001100100100001001010010011000100111"
-    "0010100000101001001010100010101100101100001011010010111000101111"
-    "0011000000110001001100100011001100110100001101010011011000110111"
-    "0011100000111001001110100011101100111100001111010011111000111111"
-    "0100000001000001010000100100001101000100010001010100011001000111"
-    "0100100001001001010010100100101101001100010011010100111001001111"
-    "0101000001010001010100100101001101010100010101010101011001010111"
-    "0101100001011001010110100101101101011100010111010101111001011111"
-    "0110000001100001011000100110001101100100011001010110011001100111"
-    "0110100001101001011010100110101101101100011011010110111001101111"
-    "0111000001110001011100100111001101110100011101010111011001110111"
-    "0111100001111001011110100111101101111100011111010111111001111111"
-    "1000000010000001100000101000001110000100100001011000011010000111"
-    "1000100010001001100010101000101110001100100011011000111010001111"
-    "1001000010010001100100101001001110010100100101011001011010010111"
-    "1001100010011001100110101001101110011100100111011001111010011111"
-    "1010000010100001101000101010001110100100101001011010011010100111"
-    "1010100010101001101010101010101110101100101011011010111010101111"
-    "1011000010110001101100101011001110110100101101011011011010110111"
-    "1011100010111001101110101011101110111100101111011011111010111111"
-    "1100000011000001110000101100001111000100110001011100011011000111"
-    "1100100011001001110010101100101111001100110011011100111011001111"
-    "1101000011010001110100101101001111010100110101011101011011010111"
-    "1101100011011001110110101101101111011100110111011101111011011111"
-    "1110000011100001111000101110001111100100111001011110011011100111"
-    "1110100011101001111010101110101111101100111011011110111011101111"
-    "1111000011110001111100101111001111110100111101011111011011110111"
-    "1111100011111001111110101111101111111100111111011111111011111111";
-
-inline void writeBinByte(UInt8 byte, void * out)
+/// Converts a hex representation with leading zeros back to an integer value.
+/// The function supports native integer types, wide::integer, CityHash_v1_0_2::uint128.
+template <typename T>
+constexpr T unhexUInt(const char * data)
 {
-    memcpy(out, &bin_byte_to_char_table[static_cast<size_t>(byte) * 8], 8);
+    return impl::HexConversion<T>::unhex(data);
 }
 
-/// Produces hex representation of an unsigned int with leading zeros (for checksums)
-template <typename TUInt>
-inline void writeHexUIntImpl(TUInt uint_, char * out, std::string_view table)
-{
-    union
-    {
-        TUInt value;
-        UInt8 uint8[sizeof(TUInt)];
-    };
-
-    value = uint_;
-
-    for (size_t i = 0; i < sizeof(TUInt); ++i)
-    {
-        if constexpr (std::endian::native == std::endian::little)
-            memcpy(out + i * 2, &table[static_cast<size_t>(uint8[sizeof(TUInt) - 1 - i]) * 2], 2);
-        else
-            memcpy(out + i * 2, &table[static_cast<size_t>(uint8[i]) * 2], 2);
-    }
-}
-
-template <typename TUInt>
-inline void writeHexUIntUppercase(TUInt uint_, char * out)
-{
-    writeHexUIntImpl(uint_, out, hex_byte_to_char_uppercase_table);
-}
-
-template <typename TUInt>
-inline void writeHexUIntLowercase(TUInt uint_, char * out)
-{
-    writeHexUIntImpl(uint_, out, hex_byte_to_char_lowercase_table);
-}
-
-template <typename TUInt>
-std::string getHexUIntUppercase(TUInt uint_)
-{
-    std::string res(sizeof(TUInt) * 2, '\0');
-    writeHexUIntUppercase(uint_, res.data());
-    return res;
-}
-
-template <typename TUInt>
-std::string getHexUIntLowercase(TUInt uint_)
-{
-    std::string res(sizeof(TUInt) * 2, '\0');
-    writeHexUIntLowercase(uint_, res.data());
-    return res;
-}
-
-/// Maps 0..9, A..F, a..f to 0..15. Other chars are mapped to implementation specific value.
-
-constexpr inline std::string_view hex_char_to_digit_table
-    = {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" //0-9
-       "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //A-Z
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //a-z
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
-       "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff",
-       256};
-
+/// Converts a hexadecimal digit '0'..'f' or '0'..'F' to UInt8.
 constexpr UInt8 unhex(char c)
 {
-    return hex_char_to_digit_table[static_cast<UInt8>(c)];
+    return impl::unhexDigit(c);
 }
 
+/// Converts two hexadecimal digits to UInt8.
 constexpr UInt8 unhex2(const char * data)
 {
-    return static_cast<UInt8>(unhex(data[0])) * 0x10 + static_cast<UInt8>(unhex(data[1]));
+    return unhexUInt<UInt8>(data);
 }
 
+/// Converts four hexadecimal digits to UInt16.
 constexpr UInt16 unhex4(const char * data)
 {
-    return static_cast<UInt16>(unhex(data[0])) * 0x1000 + static_cast<UInt16>(unhex(data[1])) * 0x100
-        + static_cast<UInt16>(unhex(data[2])) * 0x10 + static_cast<UInt16>(unhex(data[3]));
+    return unhexUInt<UInt16>(data);
 }
 
-template <typename TUInt>
-constexpr TUInt unhexUInt(const char * data)
+/// Produces a binary representation of a single byte.
+inline void writeBinByte(UInt8 byte, void * out)
 {
-    TUInt res = 0;
-    if constexpr ((sizeof(TUInt) <= 8) || ((sizeof(TUInt) % 8) != 0))
-    {
-        for (size_t i = 0; i < sizeof(TUInt) * 2; ++i, ++data)
-        {
-            res <<= 4;
-            res += unhex(*data);
-        }
-    }
-    else
-    {
-        for (size_t i = 0; i < sizeof(TUInt) / 8; ++i, data += 16)
-        {
-            res <<= 64;
-            res += unhexUInt<UInt64>(data);
-        }
-    }
-    return res;
+    memcpy(out, &impl::bin_byte_to_char_table[static_cast<size_t>(byte) * 8], 8);
 }

base/poco/Foundation/CMakeLists.txt

@@ -87,7 +87,6 @@ set (SRCS
     src/LoggingRegistry.cpp
     src/LogStream.cpp
     src/MD5Engine.cpp
-    src/MemoryPool.cpp
     src/MemoryStream.cpp
     src/Message.cpp
     src/Mutex.cpp

base/poco/Foundation/include/Poco/MemoryPool.h

@@ -1,116 +0,0 @@
-//
-// MemoryPool.h
-//
-// Library: Foundation
-// Package: Core
-// Module:  MemoryPool
-//
-// Definition of the MemoryPool class.
-//
-// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
-// and Contributors.
-//
-// SPDX-License-Identifier:	BSL-1.0
-//
-
-
-#ifndef Foundation_MemoryPool_INCLUDED
-#define Foundation_MemoryPool_INCLUDED
-
-
-#include <cstddef>
-#include <vector>
-#include "Poco/Foundation.h"
-#include "Poco/Mutex.h"
-
-
-namespace Poco
-{
-
-
-class Foundation_API MemoryPool
-/// A simple pool for fixed-size memory blocks.
-///
-/// The main purpose of this class is to speed-up
-/// memory allocations, as well as to reduce memory
-/// fragmentation in situations where the same blocks
-/// are allocated all over again, such as in server
-/// applications.
-///
-/// All allocated blocks are retained for future use.
-/// A limit on the number of blocks can be specified.
-/// Blocks can be preallocated.
-{
-public:
-    MemoryPool(std::size_t blockSize, int preAlloc = 0, int maxAlloc = 0);
-    /// Creates a MemoryPool for blocks with the given blockSize.
-    /// The number of blocks given in preAlloc are preallocated.
-
-    ~MemoryPool();
-
-    void * get();
-    /// Returns a memory block. If there are no more blocks
-    /// in the pool, a new block will be allocated.
-    ///
-    /// If maxAlloc blocks are already allocated, an
-    /// OutOfMemoryException is thrown.
-
-    void release(void * ptr);
-    /// Releases a memory block and returns it to the pool.
-
-    std::size_t blockSize() const;
-    /// Returns the block size.
-
-    int allocated() const;
-    /// Returns the number of allocated blocks.
-
-    int available() const;
-    /// Returns the number of available blocks in the pool.
-
-private:
-    MemoryPool();
-    MemoryPool(const MemoryPool &);
-    MemoryPool & operator=(const MemoryPool &);
-
-    void clear();
-
-    enum
-    {
-        BLOCK_RESERVE = 128
-    };
-
-    typedef std::vector<char *> BlockVec;
-
-    std::size_t _blockSize;
-    int _maxAlloc;
-    int _allocated;
-    BlockVec _blocks;
-    FastMutex _mutex;
-};
-
-
-//
-// inlines
-//
-inline std::size_t MemoryPool::blockSize() const
-{
-    return _blockSize;
-}
-
-
-inline int MemoryPool::allocated() const
-{
-    return _allocated;
-}
-
-
-inline int MemoryPool::available() const
-{
-    return (int)_blocks.size();
-}
-
-
-} // namespace Poco
-
-
-#endif // Foundation_MemoryPool_INCLUDED

base/poco/Foundation/src/MemoryPool.cpp

@@ -1,105 +0,0 @@
-//
-// MemoryPool.cpp
-//
-// Library: Foundation
-// Package: Core
-// Module:  MemoryPool
-//
-// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
-// and Contributors.
-//
-// SPDX-License-Identifier:	BSL-1.0
-//
-
-
-#include "Poco/MemoryPool.h"
-#include "Poco/Exception.h"
-
-
-namespace Poco {
-
-
-MemoryPool::MemoryPool(std::size_t blockSize, int preAlloc, int maxAlloc):
-	_blockSize(blockSize),
-	_maxAlloc(maxAlloc),
-	_allocated(preAlloc)
-{
-	poco_assert (maxAlloc == 0 || maxAlloc >= preAlloc);
-	poco_assert (preAlloc >= 0 && maxAlloc >= 0);
-
-	int r = BLOCK_RESERVE;
-	if (preAlloc > r)
-		r = preAlloc;
-	if (maxAlloc > 0 && maxAlloc < r)
-		r = maxAlloc;
-	_blocks.reserve(r);
-	
-	try
-	{
-		for (int i = 0; i < preAlloc; ++i)
-		{
-			_blocks.push_back(new char[_blockSize]);
-		}
-	}
-	catch (...)
-	{
-		clear();
-		throw;
-	}
-}
-
-	
-MemoryPool::~MemoryPool()
-{
-	clear();
-}
-
-
-void MemoryPool::clear()
-{
-	for (BlockVec::iterator it = _blocks.begin(); it != _blocks.end(); ++it)
-	{
-		delete [] *it;
-	}
-	_blocks.clear();
-}
-
-
-void* MemoryPool::get()
-{
-	FastMutex::ScopedLock lock(_mutex);
-	
-	if (_blocks.empty())
-	{
-		if (_maxAlloc == 0 || _allocated < _maxAlloc)
-		{
-			++_allocated;
-			return new char[_blockSize];
-		}
-		else throw OutOfMemoryException("MemoryPool exhausted");
-	}
-	else
-	{
-		char* ptr = _blocks.back();
-		_blocks.pop_back();
-		return ptr;
-	}
-}
-
-	
-void MemoryPool::release(void* ptr)
-{
-	FastMutex::ScopedLock lock(_mutex);
-	
-	try
-	{
-		_blocks.push_back(reinterpret_cast<char*>(ptr));
-	}
-	catch (...)
-	{
-		delete [] reinterpret_cast<char*>(ptr);
-	}
-}
-
-
-} // namespace Poco

base/poco/Net/include/Poco/Net/HTTPBasicStreamBuf.h

@@ -19,7 +19,6 @@
 
 
 #include "Poco/BufferedStreamBuf.h"
-#include "Poco/Net/HTTPBufferAllocator.h"
 #include "Poco/Net/Net.h"
 
 
@@ -27,9 +26,9 @@ namespace Poco
 {
 namespace Net
 {
+    constexpr size_t HTTP_DEFAULT_BUFFER_SIZE = 8 * 1024;
 
-
-    typedef Poco::BasicBufferedStreamBuf<char, std::char_traits<char>, HTTPBufferAllocator> HTTPBasicStreamBuf;
+    typedef Poco::BasicBufferedStreamBuf<char, std::char_traits<char>> HTTPBasicStreamBuf;
 
 
 }

base/poco/Net/include/Poco/Net/HTTPBufferAllocator.h

@@ -1,53 +0,0 @@
-//
-// HTTPBufferAllocator.h
-//
-// Library: Net
-// Package: HTTP
-// Module:  HTTPBufferAllocator
-//
-// Definition of the HTTPBufferAllocator class.
-//
-// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
-// and Contributors.
-//
-// SPDX-License-Identifier:	BSL-1.0
-//
-
-
-#ifndef Net_HTTPBufferAllocator_INCLUDED
-#define Net_HTTPBufferAllocator_INCLUDED
-
-
-#include <ios>
-#include "Poco/MemoryPool.h"
-#include "Poco/Net/Net.h"
-
-
-namespace Poco
-{
-namespace Net
-{
-
-
-    class Net_API HTTPBufferAllocator
-    /// A BufferAllocator for HTTP streams.
-    {
-    public:
-        static char * allocate(std::streamsize size);
-        static void deallocate(char * ptr, std::streamsize size);
-
-        enum
-        {
-            BUFFER_SIZE = 128 * 1024
-        };
-
-    private:
-        static Poco::MemoryPool _pool;
-    };
-
-
-}
-} // namespace Poco::Net
-
-
-#endif // Net_HTTPBufferAllocator_INCLUDED

base/poco/Net/include/Poco/Net/HTTPChunkedStream.h

@@ -21,7 +21,6 @@
 #include <cstddef>
 #include <istream>
 #include <ostream>
-#include "Poco/MemoryPool.h"
 #include "Poco/Net/HTTPBasicStreamBuf.h"
 #include "Poco/Net/Net.h"
 
@@ -80,12 +79,6 @@ namespace Net
     public:
         HTTPChunkedInputStream(HTTPSession & session);
         ~HTTPChunkedInputStream();
-
-        void * operator new(std::size_t size);
-        void operator delete(void * ptr);
-
-    private:
-        static Poco::MemoryPool _pool;
     };
 
 
@@ -95,12 +88,6 @@ namespace Net
     public:
         HTTPChunkedOutputStream(HTTPSession & session);
         ~HTTPChunkedOutputStream();
-
-        void * operator new(std::size_t size);
-        void operator delete(void * ptr);
-
-    private:
-        static Poco::MemoryPool _pool;
     };
 
 

base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h

@@ -78,12 +78,6 @@ namespace Net
     public:
         HTTPFixedLengthInputStream(HTTPSession & session, HTTPFixedLengthStreamBuf::ContentLength length);
         ~HTTPFixedLengthInputStream();
-
-        void * operator new(std::size_t size);
-        void operator delete(void * ptr);
-
-    private:
-        static Poco::MemoryPool _pool;
     };
 
 
@@ -93,12 +87,6 @@ namespace Net
     public:
         HTTPFixedLengthOutputStream(HTTPSession & session, HTTPFixedLengthStreamBuf::ContentLength length);
         ~HTTPFixedLengthOutputStream();
-
-        void * operator new(std::size_t size);
-        void operator delete(void * ptr);
-
-    private:
-        static Poco::MemoryPool _pool;
     };
 
 

base/poco/Net/include/Poco/Net/HTTPHeaderStream.h

@@ -21,7 +21,6 @@
 #include <cstddef>
 #include <istream>
 #include <ostream>
-#include "Poco/MemoryPool.h"
 #include "Poco/Net/HTTPBasicStreamBuf.h"
 #include "Poco/Net/Net.h"
 
@@ -74,12 +73,6 @@ namespace Net
     public:
         HTTPHeaderInputStream(HTTPSession & session);
         ~HTTPHeaderInputStream();
-
-        void * operator new(std::size_t size);
-        void operator delete(void * ptr);
-
-    private:
-        static Poco::MemoryPool _pool;
     };
 
 
@@ -89,12 +82,6 @@ namespace Net
     public:
         HTTPHeaderOutputStream(HTTPSession & session);
         ~HTTPHeaderOutputStream();
-
-        void * operator new(std::size_t size);
-        void operator delete(void * ptr);
-
-    private:
-        static Poco::MemoryPool _pool;
     };
 
 

base/poco/Net/include/Poco/Net/HTTPSession.h

@@ -192,7 +192,7 @@ namespace Net
         HTTPSession & operator=(const HTTPSession &);
 
         StreamSocket _socket;
-        char * _pBuffer;
+        std::unique_ptr<char[]> _pBuffer;
         char * _pCurrent;
         char * _pEnd;
         bool _keepAlive;

base/poco/Net/include/Poco/Net/HTTPStream.h

@@ -21,7 +21,6 @@
 #include <cstddef>
 #include <istream>
 #include <ostream>
-#include "Poco/MemoryPool.h"
 #include "Poco/Net/HTTPBasicStreamBuf.h"
 #include "Poco/Net/Net.h"
 
@@ -75,12 +74,6 @@ namespace Net
     public:
         HTTPInputStream(HTTPSession & session);
         ~HTTPInputStream();
-
-        void * operator new(std::size_t size);
-        void operator delete(void * ptr);
-
-    private:
-        static Poco::MemoryPool _pool;
     };
 
 
@@ -90,12 +83,6 @@ namespace Net
     public:
         HTTPOutputStream(HTTPSession & session);
         ~HTTPOutputStream();
-
-        void * operator new(std::size_t size);
-        void operator delete(void * ptr);
-
-    private:
-        static Poco::MemoryPool _pool;
     };
 
 

base/poco/Net/src/HTTPBufferAllocator.cpp

@@ -1,44 +0,0 @@
-//
-// HTTPBufferAllocator.cpp
-//
-// Library: Net
-// Package: HTTP
-// Module:  HTTPBufferAllocator
-//
-// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
-// and Contributors.
-//
-// SPDX-License-Identifier:	BSL-1.0
-//
-
-
-#include "Poco/Net/HTTPBufferAllocator.h"
-
-
-using Poco::MemoryPool;
-
-
-namespace Poco {
-namespace Net {
-
-
-MemoryPool HTTPBufferAllocator::_pool(HTTPBufferAllocator::BUFFER_SIZE, 16);
-
-
-char* HTTPBufferAllocator::allocate(std::streamsize size)
-{
-	poco_assert_dbg (size == BUFFER_SIZE);
-
-	return reinterpret_cast<char*>(_pool.get());
-}
-
-
-void HTTPBufferAllocator::deallocate(char* ptr, std::streamsize size)
-{
-	poco_assert_dbg (size == BUFFER_SIZE);
-
-	_pool.release(ptr);
-}
-
-
-} } // namespace Poco::Net

base/poco/Net/src/HTTPChunkedStream.cpp

@@ -34,7 +34,7 @@ namespace Net {
 
 
 HTTPChunkedStreamBuf::HTTPChunkedStreamBuf(HTTPSession& session, openmode mode):
-	HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
+	HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
 	_session(session),
 	_mode(mode),
 	_chunk(0)
@@ -181,10 +181,6 @@ HTTPChunkedStreamBuf* HTTPChunkedIOS::rdbuf()
 // HTTPChunkedInputStream
 //
 
-
-Poco::MemoryPool HTTPChunkedInputStream::_pool(sizeof(HTTPChunkedInputStream));
-
-
 HTTPChunkedInputStream::HTTPChunkedInputStream(HTTPSession& session):
 	HTTPChunkedIOS(session, std::ios::in),
 	std::istream(&_buf)
@@ -196,34 +192,10 @@ HTTPChunkedInputStream::~HTTPChunkedInputStream()
 {
 }
 
-
-void* HTTPChunkedInputStream::operator new(std::size_t size)
-{
-	return _pool.get();
-}
-
-
-void HTTPChunkedInputStream::operator delete(void* ptr)
-{
-	try
-	{
-		_pool.release(ptr);
-	}
-	catch (...)
-	{
-		poco_unexpected();
-	}
-}
-
-
 //
 // HTTPChunkedOutputStream
 //
 
-
-Poco::MemoryPool HTTPChunkedOutputStream::_pool(sizeof(HTTPChunkedOutputStream));
-
-
 HTTPChunkedOutputStream::HTTPChunkedOutputStream(HTTPSession& session):
 	HTTPChunkedIOS(session, std::ios::out),
 	std::ostream(&_buf)
@@ -235,24 +207,4 @@ HTTPChunkedOutputStream::~HTTPChunkedOutputStream()
 {
 }
 
-
-void* HTTPChunkedOutputStream::operator new(std::size_t size)
-{
-	return _pool.get();
-}
-
-
-void HTTPChunkedOutputStream::operator delete(void* ptr)
-{
-	try
-	{
-		_pool.release(ptr);
-	}
-	catch (...)
-	{
-		poco_unexpected();
-	}
-}
-
-
 } } // namespace Poco::Net

base/poco/Net/src/HTTPFixedLengthStream.cpp

@@ -30,7 +30,7 @@ namespace Net {
 
 
 HTTPFixedLengthStreamBuf::HTTPFixedLengthStreamBuf(HTTPSession& session, ContentLength length, openmode mode):
-	HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
+	HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
 	_session(session),
 	_length(length),
 	_count(0)
@@ -109,9 +109,6 @@ HTTPFixedLengthStreamBuf* HTTPFixedLengthIOS::rdbuf()
 //
 
 
-Poco::MemoryPool HTTPFixedLengthInputStream::_pool(sizeof(HTTPFixedLengthInputStream));
-
-
 HTTPFixedLengthInputStream::HTTPFixedLengthInputStream(HTTPSession& session, HTTPFixedLengthStreamBuf::ContentLength length):
 	HTTPFixedLengthIOS(session, length, std::ios::in),
 	std::istream(&_buf)
@@ -124,33 +121,10 @@ HTTPFixedLengthInputStream::~HTTPFixedLengthInputStream()
 }
 
 
-void* HTTPFixedLengthInputStream::operator new(std::size_t size)
-{
-	return _pool.get();
-}
-
-
-void HTTPFixedLengthInputStream::operator delete(void* ptr)
-{
-	try
-	{
-		_pool.release(ptr);
-	}
-	catch (...)
-	{
-		poco_unexpected();
-	}
-}
-
-
 //
 // HTTPFixedLengthOutputStream
 //
 
-
-Poco::MemoryPool HTTPFixedLengthOutputStream::_pool(sizeof(HTTPFixedLengthOutputStream));
-
-
 HTTPFixedLengthOutputStream::HTTPFixedLengthOutputStream(HTTPSession& session, HTTPFixedLengthStreamBuf::ContentLength length):
 	HTTPFixedLengthIOS(session, length, std::ios::out),
 	std::ostream(&_buf)
@@ -163,23 +137,4 @@ HTTPFixedLengthOutputStream::~HTTPFixedLengthOutputStream()
 }
 
 
-void* HTTPFixedLengthOutputStream::operator new(std::size_t size)
-{
-	return _pool.get();
-}
-
-
-void HTTPFixedLengthOutputStream::operator delete(void* ptr)
-{
-	try
-	{
-		_pool.release(ptr);
-	}
-	catch (...)
-	{
-		poco_unexpected();
-	}
-}
-
-
 } } // namespace Poco::Net

base/poco/Net/src/HTTPHeaderStream.cpp

@@ -26,7 +26,7 @@ namespace Net {
 
 
 HTTPHeaderStreamBuf::HTTPHeaderStreamBuf(HTTPSession& session, openmode mode):
-	HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
+	HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
 	_session(session),
 	_end(false)
 {
@@ -101,10 +101,6 @@ HTTPHeaderStreamBuf* HTTPHeaderIOS::rdbuf()
 // HTTPHeaderInputStream
 //
 
-
-Poco::MemoryPool HTTPHeaderInputStream::_pool(sizeof(HTTPHeaderInputStream));
-
-
 HTTPHeaderInputStream::HTTPHeaderInputStream(HTTPSession& session):
 	HTTPHeaderIOS(session, std::ios::in),
 	std::istream(&_buf)
@@ -116,34 +112,10 @@ HTTPHeaderInputStream::~HTTPHeaderInputStream()
 {
 }
 
-
-void* HTTPHeaderInputStream::operator new(std::size_t size)
-{
-	return _pool.get();
-}
-
-
-void HTTPHeaderInputStream::operator delete(void* ptr)
-{
-	try
-	{
-		_pool.release(ptr);
-	}
-	catch (...)
-	{
-		poco_unexpected();
-	}
-}
-
-
 //
 // HTTPHeaderOutputStream
 //
 
-
-Poco::MemoryPool HTTPHeaderOutputStream::_pool(sizeof(HTTPHeaderOutputStream));
-
-
 HTTPHeaderOutputStream::HTTPHeaderOutputStream(HTTPSession& session):
 	HTTPHeaderIOS(session, std::ios::out),
 	std::ostream(&_buf)
@@ -155,24 +127,4 @@ HTTPHeaderOutputStream::~HTTPHeaderOutputStream()
 {
 }
 
-
-void* HTTPHeaderOutputStream::operator new(std::size_t size)
-{
-	return _pool.get();
-}
-
-
-void HTTPHeaderOutputStream::operator delete(void* ptr)
-{
-	try
-	{
-		_pool.release(ptr);
-	}
-	catch (...)
-	{
-		poco_unexpected();
-	}
-}
-
-
 } } // namespace Poco::Net

base/poco/Net/src/HTTPSession.cpp

@@ -13,8 +13,8 @@
 
 
 #include "Poco/Net/HTTPSession.h"
-#include "Poco/Net/HTTPBufferAllocator.h"
 #include "Poco/Net/NetException.h"
+#include "Poco/Net/HTTPBasicStreamBuf.h"
 #include <cstring>
 
 
@@ -68,14 +68,6 @@ HTTPSession::HTTPSession(const StreamSocket& socket, bool keepAlive):
 
 HTTPSession::~HTTPSession()
 {
-	try
-	{
-		if (_pBuffer) HTTPBufferAllocator::deallocate(_pBuffer, HTTPBufferAllocator::BUFFER_SIZE);
-	}
-	catch (...)
-	{
-		poco_unexpected();
-	}
 	try
 	{
 		close();
@@ -177,10 +169,10 @@ void HTTPSession::refill()
 {
 	if (!_pBuffer)
 	{
-		_pBuffer = HTTPBufferAllocator::allocate(HTTPBufferAllocator::BUFFER_SIZE);
+		_pBuffer = std::make_unique<char[]>(HTTP_DEFAULT_BUFFER_SIZE);
 	}
-	_pCurrent = _pEnd = _pBuffer;
-	int n = receive(_pBuffer, HTTPBufferAllocator::BUFFER_SIZE);
+	_pCurrent = _pEnd = _pBuffer.get();
+	int n = receive(_pBuffer.get(), HTTP_DEFAULT_BUFFER_SIZE);
 	_pEnd += n;
 }
 
@@ -199,7 +191,7 @@ void HTTPSession::connect(const SocketAddress& address)
 	_socket.setNoDelay(true);
 	// There may be leftover data from a previous (failed) request in the buffer,
 	// so we clear it.
-	_pCurrent = _pEnd = _pBuffer;
+	_pCurrent = _pEnd = _pBuffer.get();
 }
 
 

base/poco/Net/src/HTTPStream.cpp

@@ -26,7 +26,7 @@ namespace Net {
 
 
 HTTPStreamBuf::HTTPStreamBuf(HTTPSession& session, openmode mode):
-	HTTPBasicStreamBuf(HTTPBufferAllocator::BUFFER_SIZE, mode),
+	HTTPBasicStreamBuf(HTTP_DEFAULT_BUFFER_SIZE, mode),
 	_session(session),
 	_mode(mode)
 {
@@ -96,10 +96,6 @@ HTTPStreamBuf* HTTPIOS::rdbuf()
 // HTTPInputStream
 //
 
-
-Poco::MemoryPool HTTPInputStream::_pool(sizeof(HTTPInputStream));
-
-
 HTTPInputStream::HTTPInputStream(HTTPSession& session):
 	HTTPIOS(session, std::ios::in),
 	std::istream(&_buf)
@@ -112,33 +108,11 @@ HTTPInputStream::~HTTPInputStream()
 }
 
 
-void* HTTPInputStream::operator new(std::size_t size)
-{
-	return _pool.get();
-}
-
-
-void HTTPInputStream::operator delete(void* ptr)
-{
-	try
-	{
-		_pool.release(ptr);
-	}
-	catch (...)
-	{
-		poco_unexpected();
-	}
-}
-
-
 //
 // HTTPOutputStream
 //
 
 
-Poco::MemoryPool HTTPOutputStream::_pool(sizeof(HTTPOutputStream));
-
-
 HTTPOutputStream::HTTPOutputStream(HTTPSession& session):
 	HTTPIOS(session, std::ios::out),
 	std::ostream(&_buf)
@@ -150,24 +124,4 @@ HTTPOutputStream::~HTTPOutputStream()
 {
 }
 
-
-void* HTTPOutputStream::operator new(std::size_t size)
-{
-	return _pool.get();
-}
-
-
-void HTTPOutputStream::operator delete(void* ptr)
-{
-	try
-	{
-		_pool.release(ptr);
-	}
-	catch (...)
-	{
-		poco_unexpected();
-	}
-}
-
-
 } } // namespace Poco::Net

cmake/target.cmake

@@ -33,6 +33,19 @@ if (CMAKE_CROSSCOMPILING)
         elseif (ARCH_PPC64LE)
             set (ENABLE_GRPC OFF CACHE INTERNAL "")
             set (ENABLE_SENTRY OFF CACHE INTERNAL "")
+        elseif (ARCH_RISCV64)
+            # RISC-V support is preliminary
+            set (GLIBC_COMPATIBILITY OFF CACHE INTERNAL "")
+            set (ENABLE_LDAP OFF CACHE INTERNAL "")
+            set (OPENSSL_NO_ASM ON CACHE INTERNAL "")
+            set (ENABLE_JEMALLOC ON CACHE INTERNAL "")
+            set (ENABLE_PARQUET OFF CACHE INTERNAL "")
+            set (USE_UNWIND OFF CACHE INTERNAL "")
+            set (ENABLE_GRPC OFF CACHE INTERNAL "")
+            set (ENABLE_HDFS OFF CACHE INTERNAL "")
+            set (ENABLE_MYSQL OFF CACHE INTERNAL "")
+            # It might be ok, but we need to update 'sysroot'
+            set (ENABLE_RUST OFF CACHE INTERNAL "")
         elseif (ARCH_S390X)
             set (ENABLE_GRPC OFF CACHE INTERNAL "")
             set (ENABLE_SENTRY OFF CACHE INTERNAL "")

docker/packager/packager

@@ -138,6 +138,7 @@ def parse_env_variables(
     ARM_V80COMPAT_SUFFIX = "-aarch64-v80compat"
     FREEBSD_SUFFIX = "-freebsd"
     PPC_SUFFIX = "-ppc64le"
+    RISCV_SUFFIX = "-riscv64"
     AMD64_COMPAT_SUFFIX = "-amd64-compat"
 
     result = []
@@ -150,6 +151,7 @@ def parse_env_variables(
     is_cross_arm = compiler.endswith(ARM_SUFFIX)
     is_cross_arm_v80compat = compiler.endswith(ARM_V80COMPAT_SUFFIX)
     is_cross_ppc = compiler.endswith(PPC_SUFFIX)
+    is_cross_riscv = compiler.endswith(RISCV_SUFFIX)
     is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
     is_amd64_compat = compiler.endswith(AMD64_COMPAT_SUFFIX)
 
@@ -206,6 +208,11 @@ def parse_env_variables(
         cmake_flags.append(
             "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake"
         )
+    elif is_cross_riscv:
+        cc = compiler[: -len(RISCV_SUFFIX)]
+        cmake_flags.append(
+            "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-riscv64.cmake"
+        )
     elif is_amd64_compat:
         cc = compiler[: -len(AMD64_COMPAT_SUFFIX)]
         result.append("DEB_ARCH=amd64")
@@ -370,6 +377,7 @@ def parse_args() -> argparse.Namespace:
             "clang-16-aarch64",
             "clang-16-aarch64-v80compat",
             "clang-16-ppc64le",
+            "clang-16-riscv64",
             "clang-16-amd64-compat",
             "clang-16-freebsd",
         ),

docker/test/integration/runner/Dockerfile

@@ -47,11 +47,13 @@ ENV TZ=Etc/UTC
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 ENV DOCKER_CHANNEL stable
+# Unpin the docker version after the release 24.0.3 is released
+# https://github.com/moby/moby/issues/45770#issuecomment-1618255130
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
     && add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" \
     && apt-get update \
     && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
-        docker-ce \
+        docker-ce='5:23.*' \
     && rm -rf \
         /var/lib/apt/lists/* \
         /var/cache/debconf \

docker/test/style/Dockerfile

@@ -18,7 +18,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
     python3-pip \
     shellcheck \
     yamllint \
-    && pip3 install black==23.1.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \
+    && pip3 install black==23.1.0 boto3 codespell==2.2.1 mypy==1.3.0 PyGithub unidiff pylint==2.6.2 \
     && apt-get clean \
     && rm -rf /root/.cache/pip
 

docker/test/upgrade/run.sh

@@ -67,6 +67,13 @@ start
 stop
 mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.initial.log
 
+# Start server from previous release
+# Let's enable S3 storage by default
+export USE_S3_STORAGE_FOR_MERGE_TREE=1
+# Previous version may not be ready for fault injections
+export ZOOKEEPER_FAULT_INJECTION=0
+configure
+
 # force_sync=false doesn't work correctly on some older versions
 sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
   | sed "s|<force_sync>false</force_sync>|<force_sync>true</force_sync>|" \
@@ -76,17 +83,11 @@ sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-serv
 # But we still need default disk because some tables loaded only into it
 sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
   | sed "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" \
-  > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp    mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
+  > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp
+mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 
-# Start server from previous release
-# Let's enable S3 storage by default
-export USE_S3_STORAGE_FOR_MERGE_TREE=1
-# Previous version may not be ready for fault injections
-export ZOOKEEPER_FAULT_INJECTION=0
-configure
-
 # it contains some new settings, but we can safely remove it
 rm /etc/clickhouse-server/config.d/merge_tree.xml
 rm /etc/clickhouse-server/users.d/nonconst_timezone.xml

docker/test/util/Dockerfile

@@ -94,7 +94,10 @@ RUN mkdir /tmp/ccache \
     && rm -rf /tmp/ccache
 
 ARG TARGETARCH
-ARG SCCACHE_VERSION=v0.4.1
+ARG SCCACHE_VERSION=v0.5.4
+ENV SCCACHE_IGNORE_SERVER_IO_ERROR=1
+# sccache requires a value for the region. So by default we use The Default Region
+ENV SCCACHE_REGION=us-east-1
 RUN arch=${TARGETARCH:-amd64} \
   && case $arch in \
     amd64) rarch=x86_64 ;; \

docs/_description_templates/template-data-type.md

@@ -0,0 +1,29 @@
+---
+toc_priority:
+toc_title:
+---
+
+# data_type_name {#data_type-name}
+
+Description.
+
+**Parameters** (Optional)
+
+-   `x` — Description. [Type name](relative/path/to/type/dscr.md#type).
+-   `y` — Description. [Type name](relative/path/to/type/dscr.md#type).
+
+**Examples**
+
+```sql
+
+```
+
+## Additional Info {#additional-info} (Optional)
+
+The name of an additional section can be any, for example, **Usage**.
+
+**See Also** (Optional)
+
+-   [link](#)
+
+[Original article](https://clickhouse.com/docs/en/data-types/<data-type-name>/) <!--hide-->

docs/_description_templates/template-engine.md

@@ -0,0 +1,63 @@
+# EngineName {#enginename}
+
+-   What the Database/Table engine does.
+-   Relations with other engines if they exist.
+
+## Creating a Database {#creating-a-database}
+``` sql
+    CREATE DATABASE ...
+```
+or
+
+## Creating a Table {#creating-a-table}
+``` sql
+    CREATE TABLE ...
+```
+
+**Engine Parameters**
+
+**Query Clauses** (for Table engines only)
+
+## Virtual columns {#virtual-columns} (for Table engines only)
+
+List and virtual columns with description, if they exist.
+
+## Data Types Support {#data_types-support} (for Database engines only)
+
+|  EngineName           | ClickHouse                         |
+|-----------------------|------------------------------------|
+| NativeDataTypeName    | [ClickHouseDataTypeName](link#)    |
+
+
+## Specifics and recommendations {#specifics-and-recommendations}
+
+Algorithms
+Specifics of read and write processes
+Examples of tasks
+Recommendations for usage
+Specifics of data storage
+
+## Usage Example {#usage-example}
+
+The example must show usage and use cases. The following text contains the recommended parts of this section.
+
+Input table:
+
+``` text
+```
+
+Query:
+
+``` sql
+```
+
+Result:
+
+``` text
+```
+
+Follow up with any text to clarify the example.
+
+**See Also**
+
+-   [link](#)

docs/_description_templates/template-function.md

@@ -0,0 +1,51 @@
+## functionName {#functionname-in-lower-case}
+
+Short description.
+
+**Syntax** (without SELECT)
+
+``` sql
+<function syntax>
+```
+
+Alias: `<alias name>`. (Optional)
+
+More text (Optional).
+
+**Arguments** (Optional)
+
+-   `x` — Description. Optional (only for optional arguments). Possible values: <values list>. Default value: <value>. [Type name](relative/path/to/type/dscr.md#type).
+-   `y` — Description. Optional (only for optional arguments). Possible values: <values list>.Default value: <value>. [Type name](relative/path/to/type/dscr.md#type).
+
+**Parameters** (Optional, only for parametric aggregate functions)
+
+-   `z` — Description. Optional (only for optional parameters). Possible values: <values list>. Default value: <value>. [Type name](relative/path/to/type/dscr.md#type).
+
+**Returned value(s)**
+
+-   Returned values list.
+
+Type: [Type name](relative/path/to/type/dscr.md#type).
+
+**Example**
+
+The example must show usage and/or a use cases. The following text contains recommended parts of an example.
+
+Input table (Optional):
+
+``` text
+```
+
+Query:
+
+``` sql
+```
+
+Result:
+
+``` text
+```
+
+**See Also** (Optional)
+
+-   [link](#)

docs/_description_templates/template-server-setting.md

@@ -0,0 +1,33 @@
+## server_setting_name {#server_setting_name}
+
+Description.
+
+Describe what is configured in this section of settings.
+
+Possible value: ...
+
+Default value: ...
+
+**Settings** (Optional)
+
+If the section contains several settings, list them here. Specify possible values and default values:
+
+-   setting_1 — Description.
+-   setting_2 — Description.
+
+**Example**
+
+```xml
+<server_setting_name>
+    <setting_1> ... </setting_1>
+    <setting_2> ... </setting_2>
+</server_setting_name>
+```
+
+**Additional Info** (Optional)
+
+The name of an additional section can be any, for example, **Usage**.
+
+**See Also** (Optional)
+
+-   [link](#)

docs/_description_templates/template-setting.md

@@ -0,0 +1,27 @@
+## setting_name {#setting_name}
+
+Description.
+
+For the switch setting, use the typical phrase: “Enables or disables something …”.
+
+Possible values:
+
+*For switcher setting:*
+
+-   0 — Disabled.
+-   1 — Enabled.
+
+*For another setting (typical phrases):*
+
+-   Positive integer.
+-   0 — Disabled or unlimited or something else.
+
+Default value: `value`.
+
+**Additional Info** (Optional)
+
+The name of an additional section can be any, for example, **Usage**.
+
+**See Also** (Optional)
+
+-   [link](#)

docs/_description_templates/template-statement.md

@@ -0,0 +1,24 @@
+# Statement name (for example, SHOW USER) {#statement-name-in-lower-case}
+
+Brief description of what the statement does.
+
+**Syntax**
+
+```sql
+Syntax of the statement.
+```
+
+## Other necessary sections of the description (Optional) {#anchor}
+
+Examples of descriptions with a complicated structure:
+
+- https://clickhouse.com/docs/en/sql-reference/statements/grant/
+- https://clickhouse.com/docs/en/sql-reference/statements/revoke/
+- https://clickhouse.com/docs/en/sql-reference/statements/select/join/
+
+
+**See Also** (Optional)
+
+Links to related topics as a list.
+
+-   [link](#)

docs/_description_templates/template-system-table.md

@@ -0,0 +1,25 @@
+# system.table_name {#system-tables_table-name}
+
+Description.
+
+Columns:
+
+-   `column_name` ([data_type_name](path/to/data_type.md)) — Description.
+
+**Example**
+
+Query:
+
+``` sql
+SELECT * FROM system.table_name
+```
+
+Result:
+
+``` text
+Some output. It shouldn't be too long.
+```
+
+**See Also**
+
+-   [Article name](path/to/article_name.md) — Some words about referenced information.

docs/_includes/install/universal.sh

@@ -33,6 +33,9 @@ then
     elif [ "${ARCH}" = "powerpc64le" -o "${ARCH}" = "ppc64le" ]
     then
         DIR="powerpc64le"
+    elif [ "${ARCH}" = "riscv64" ]
+    then
+        DIR="riscv64"
     fi
 elif [ "${OS}" = "FreeBSD" ]
 then

docs/en/engines/table-engines/integrations/odbc.md

@@ -54,7 +54,7 @@ $ sudo mysql
 
 ``` sql
 mysql> CREATE USER 'clickhouse'@'localhost' IDENTIFIED BY 'clickhouse';
-mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'clickhouse' WITH GRANT OPTION;
+mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'localhost' WITH GRANT OPTION;
 ```
 
 Then configure the connection in `/etc/odbc.ini`.
@@ -66,7 +66,7 @@ DRIVER = /usr/local/lib/libmyodbc5w.so
 SERVER = 127.0.0.1
 PORT = 3306
 DATABASE = test
-USERNAME = clickhouse
+USER = clickhouse
 PASSWORD = clickhouse
 ```
 
@@ -83,6 +83,9 @@ $ isql -v mysqlconn
 Table in MySQL:
 
 ``` text
+mysql> CREATE DATABASE test;
+Query OK, 1 row affected (0,01 sec)
+
 mysql> CREATE TABLE `test`.`test` (
     ->   `int_id` INT NOT NULL AUTO_INCREMENT,
     ->   `int_nullable` INT NULL DEFAULT NULL,
@@ -91,10 +94,10 @@ mysql> CREATE TABLE `test`.`test` (
     ->   PRIMARY KEY (`int_id`));
 Query OK, 0 rows affected (0,09 sec)
 
-mysql> insert into test (`int_id`, `float`) VALUES (1,2);
+mysql> insert into test.test (`int_id`, `float`) VALUES (1,2);
 Query OK, 1 row affected (0,00 sec)
 
-mysql> select * from test;
+mysql> select * from test.test;
 +------+----------+-----+----------+
 | int_id | int_nullable | float | float_nullable |
 +------+----------+-----+----------+

docs/en/getting-started/install.md

@@ -378,6 +378,10 @@ request](https://github.com/ClickHouse/ClickHouse/commits/master) and find CI ch
 https://s3.amazonaws.com/clickhouse/builds/PRs/.../.../binary_aarch64_v80compat/clickhouse". You can then click the link to download the
 build.
 
+### macOS-only: Install with Homebrew
+
+To install ClickHouse using the popular `brew` package manager, follow the instructions listed in the [ClickHouse Homebrew tap](https://github.com/ClickHouse/homebrew-clickhouse).
+
 ## Launch {#launch}
 
 To start the server as a daemon, run:

docs/en/operations/settings/settings.md

@@ -3201,6 +3201,40 @@ ENGINE = Log
 └──────────────────────────────────────────────────────────────────────────┘
 ```
 
+## default_temporary_table_engine {#default_temporary_table_engine}
+
+Same as [default_table_engine](#default_table_engine) but for temporary tables.
+
+Default value: `Memory`.
+
+In this example, any new temporary table that does not specify an `Engine` will use the `Log` table engine:
+
+Query:
+
+```sql
+SET default_temporary_table_engine = 'Log';
+
+CREATE TEMPORARY TABLE my_table (
+    x UInt32,
+    y UInt32
+);
+
+SHOW CREATE TEMPORARY TABLE my_table;
+```
+
+Result:
+
+```response
+┌─statement────────────────────────────────────────────────────────────────┐
+│ CREATE TEMPORARY TABLE default.my_table
+(
+    `x` UInt32,
+    `y` UInt32
+)
+ENGINE = Log
+└──────────────────────────────────────────────────────────────────────────┘
+```
+
 ## data_type_default_nullable {#data_type_default_nullable}
 
 Allows data types without explicit modifiers [NULL or NOT NULL](../../sql-reference/statements/create/table.md/#null-modifiers) in column definition will be [Nullable](../../sql-reference/data-types/nullable.md/#data_type-nullable).

docs/en/operations/system-tables/asynchronous_metric_log.md

@@ -9,7 +9,6 @@ Columns:
 
 - `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
 - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
-- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds resolution.
 - `name` ([String](../../sql-reference/data-types/string.md)) — Metric name.
 - `value` ([Float64](../../sql-reference/data-types/float.md)) — Metric value.
 
@@ -20,18 +19,18 @@ SELECT * FROM system.asynchronous_metric_log LIMIT 10
 ```
 
 ``` text
-┌─event_date─┬──────────event_time─┬────event_time_microseconds─┬─name─────────────────────────────────────┬─────value─┐
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ CPUFrequencyMHz_0                        │    2120.9 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pmuzzy               │       743 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pdirty               │     26288 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.run_intervals │         0 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.num_runs      │         0 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.retained                        │  60694528 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.mapped                          │ 303161344 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.resident                        │ 260931584 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.metadata                        │  12079488 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.allocated                       │ 133756128 │
-└────────────┴─────────────────────┴────────────────────────────┴──────────────────────────────────────────┴───────────┘
+┌─event_date─┬──────────event_time─┬─name─────────────────────────────────────┬─────value─┐
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ CPUFrequencyMHz_0                        │    2120.9 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.arenas.all.pmuzzy               │       743 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.arenas.all.pdirty               │     26288 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.background_thread.run_intervals │         0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.background_thread.num_runs      │         0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.retained                        │  60694528 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.mapped                          │ 303161344 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.resident                        │ 260931584 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.metadata                        │  12079488 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.allocated                       │ 133756128 │
+└────────────┴─────────────────────┴──────────────────────────────────────────┴───────────┘
 ```
 
 **See Also**

docs/en/operations/system-tables/index.md

@@ -13,6 +13,7 @@ System tables provide information about:
 
 - Server states, processes, and environment.
 - Server’s internal processes.
+- Options used when the ClickHouse binary was built.
 
 System tables:
 

docs/en/sql-reference/statements/show.md

@@ -283,7 +283,7 @@ The optional keyword `EXTENDED` currently has no effect, it only exists for MySQ
 
 `SHOW INDEX` produces a result table with the following structure:
 - table - The name of the table (String)
-- non_unique - 0 if the index can contain duplicates, 1 otherwise (UInt8)
+- non_unique - 0 if the index cannot contain duplicates, 1 otherwise (UInt8)
 - key_name - The name of the index, `PRIMARY` if the index is a primary key index (String)
 - seq_in_index - Currently unused
 - column_name - Currently unused

docs/ru/operations/system-tables/asynchronous_metric_log.md

@@ -8,7 +8,6 @@ slug: /ru/operations/system-tables/asynchronous_metric_log
 Столбцы:
 -   `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата события.
 -   `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время события.
--   `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время события в микросекундах.
 -   `name` ([String](../../sql-reference/data-types/string.md)) — название метрики.
 -   `value` ([Float64](../../sql-reference/data-types/float.md)) — значение метрики.
 

docs/zh/operations/system-tables/asynchronous_metric_log.md

@@ -8,7 +8,6 @@ slug: /zh/operations/system-tables/asynchronous_metric_log
 列：
 -   `event_date` ([Date](../../sql-reference/data-types/date.md)) — 事件日期。
 -   `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — 事件时间。
--   `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — 事件时间(微秒)。
 -   `name` ([String](../../sql-reference/data-types/string.md)) — 指标名。
 -   `value` ([Float64](../../sql-reference/data-types/float.md)) — 指标值。
 
@@ -17,18 +16,18 @@ slug: /zh/operations/system-tables/asynchronous_metric_log
 SELECT * FROM system.asynchronous_metric_log LIMIT 10
 ```
 ``` text
-┌─event_date─┬──────────event_time─┬────event_time_microseconds─┬─name─────────────────────────────────────┬─────value─┐
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ CPUFrequencyMHz_0                        │    2120.9 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pmuzzy               │       743 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pdirty               │     26288 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.run_intervals │         0 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.num_runs      │         0 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.retained                        │  60694528 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.mapped                          │ 303161344 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.resident                        │ 260931584 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.metadata                        │  12079488 │
-│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.allocated                       │ 133756128 │
-└────────────┴─────────────────────┴────────────────────────────┴──────────────────────────────────────────┴───────────┘
+┌─event_date─┬──────────event_time─┬─name─────────────────────────────────────┬─────value─┐
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ CPUFrequencyMHz_0                        │    2120.9 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.arenas.all.pmuzzy               │       743 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.arenas.all.pdirty               │     26288 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.background_thread.run_intervals │         0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.background_thread.num_runs      │         0 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.retained                        │  60694528 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.mapped                          │ 303161344 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.resident                        │ 260931584 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.metadata                        │  12079488 │
+│ 2020-09-05 │ 2020-09-05 15:56:30 │ jemalloc.allocated                       │ 133756128 │
+└────────────┴─────────────────────┴──────────────────────────────────────────┴───────────┘
 ```
 
 **另请参阅**

docs/zh/sql-reference/functions/functions-for-nulls.md

@@ -192,7 +192,7 @@ SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook
 **返回值**
 
 -   如果`x`不为`NULL`，返回非`Nullable`类型的原始值。
--   如果`x`为`NULL`，返回对应非`Nullable`类型的默认值。
+-   如果`x`为`NULL`，则返回任意值。
 
 **示例**
 

programs/disks/CommandCopy.cpp

@@ -59,7 +59,7 @@ public:
         String relative_path_from = validatePathAndGetAsRelative(path_from);
         String relative_path_to = validatePathAndGetAsRelative(path_to);
 
-        disk_from->copy(relative_path_from, disk_to, relative_path_to);
+        disk_from->copyDirectoryContent(relative_path_from, disk_to, relative_path_to);
     }
 };
 }

programs/keeper-converter/KeeperConverter.cpp

@@ -42,7 +42,7 @@ int mainEntryClickHouseKeeperConverter(int argc, char ** argv)
     {
         auto keeper_context = std::make_shared<KeeperContext>(true);
         keeper_context->setDigestEnabled(true);
-        keeper_context->setSnapshotDisk(std::make_shared<DiskLocal>("Keeper-snapshots", options["output-dir"].as<std::string>(), 0));
+        keeper_context->setSnapshotDisk(std::make_shared<DiskLocal>("Keeper-snapshots", options["output-dir"].as<std::string>()));
 
         DB::KeeperStorage storage(/* tick_time_ms */ 500, /* superdigest */ "", keeper_context, /* initialize_system_nodes */ false);
 

programs/local/LocalServer.cpp

@@ -8,7 +8,9 @@
 #include <Poco/Logger.h>
 #include <Poco/NullChannel.h>
 #include <Poco/SimpleFileChannel.h>
+#include <Databases/DatabaseFilesystem.h>
 #include <Databases/DatabaseMemory.h>
+#include <Databases/DatabasesOverlay.h>
 #include <Storages/System/attachSystemTables.h>
 #include <Storages/System/attachInformationSchemaTables.h>
 #include <Interpreters/DatabaseCatalog.h>
@@ -50,6 +52,8 @@
 #include <base/argsToConfig.h>
 #include <filesystem>
 
+#include "config.h"
+
 #if defined(FUZZING_MODE)
     #include <Functions/getFuzzerData.h>
 #endif
@@ -170,6 +174,13 @@ static DatabasePtr createMemoryDatabaseIfNotExists(ContextPtr context, const Str
     return system_database;
 }
 
+static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, ContextPtr context_)
+{
+    auto databaseCombiner = std::make_shared<DatabasesOverlay>(name_, context_);
+    databaseCombiner->registerNextDatabase(std::make_shared<DatabaseFilesystem>(name_, "", context_));
+    databaseCombiner->registerNextDatabase(std::make_shared<DatabaseMemory>(name_, context_));
+    return databaseCombiner;
+}
 
 /// If path is specified and not empty, will try to setup server environment and load existing metadata
 void LocalServer::tryInitPath()
@@ -669,7 +680,7 @@ void LocalServer::processConfig()
       *  if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons.
       */
     std::string default_database = config().getString("default_database", "_local");
-    DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database, global_context));
+    DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context));
     global_context->setCurrentDatabase(default_database);
     applyCmdOptions(global_context);
 

programs/server/Server.cpp

@@ -1581,6 +1581,15 @@ try
         /// After attaching system databases we can initialize system log.
         global_context->initializeSystemLogs();
         global_context->setSystemZooKeeperLogAfterInitializationIfNeeded();
+        /// Build loggers before tables startup to make log messages from tables
+        /// attach available in system.text_log
+        {
+            String level_str = config().getString("text_log.level", "");
+            int level = level_str.empty() ? INT_MAX : Poco::Logger::parseLevel(level_str);
+            setTextLog(global_context->getTextLog(), level);
+
+            buildLoggers(config(), logger());
+        }
         /// After the system database is created, attach virtual system tables (in addition to query_log and part_log)
         attachSystemTablesServer(global_context, *database_catalog.getSystemDatabase(), has_zookeeper);
         attachInformationSchema(global_context, *database_catalog.getDatabase(DatabaseCatalog::INFORMATION_SCHEMA));
@@ -1707,14 +1716,6 @@ try
         /// Must be done after initialization of `servers`, because async_metrics will access `servers` variable from its thread.
         async_metrics.start();
 
-        {
-            String level_str = config().getString("text_log.level", "");
-            int level = level_str.empty() ? INT_MAX : Poco::Logger::parseLevel(level_str);
-            setTextLog(global_context->getTextLog(), level);
-        }
-
-        buildLoggers(config(), logger());
-
         main_config_reloader->start();
         access_control.startPeriodicReloading();
 

programs/server/dashboard.html

@@ -12,7 +12,8 @@
             --chart-background: white;
             --shadow-color: rgba(0, 0, 0, 0.25);
             --input-shadow-color: rgba(0, 255, 0, 1);
-            --error-color: white;
+            --error-color: red;
+            --auth-error-color: white;
             --legend-background: rgba(255, 255, 255, 0.75);
             --title-color: #666;
             --text-color: black;
@@ -258,7 +259,7 @@
             width: 60%;
             padding: .5rem;
 
-            color: var(--error-color);
+            color: var(--auth-error-color);
 
             display: flex;
             flex-flow: row nowrap;
@@ -906,9 +907,9 @@ async function draw(idx, chart, url_params, query) {
 
     if (error) {
         const errorMatch = errorMessages.find(({ regex }) => error.match(regex))
-        if (errorMatch) {
-            const match = error.match(errorMatch.regex)
-            const message = errorMatch.messageFunc(match)
+        const match = error.match(errorMatch.regex)
+        const message = errorMatch.messageFunc(match)
+        if (message) {
             const authError = new Error(message)
             throw authError
         }
@@ -930,7 +931,7 @@ async function draw(idx, chart, url_params, query) {
     let title_div = chart.querySelector('.title');
     if (error) {
         error_div.firstChild.data = error;
-        title_div.style.display = 'none';
+        title_div.style.display = 'none'; 
         error_div.style.display = 'block';
         return false;
     } else {
@@ -1019,13 +1020,15 @@ async function drawAll() {
             firstLoad = false;
         } else {
             enableReloadButton();
+            enableRunButton();
         }
-        if (!results.includes(false)) {
+        if (results.includes(true)) {
             const element = document.querySelector('.inputs');
             element.classList.remove('unconnected');
             const add = document.querySelector('#add');
             add.style.display = 'block';
-        } else {
+        } 
+        else {
             const charts = document.querySelector('#charts')
             charts.style.height = '0px';
         }
@@ -1050,6 +1053,13 @@ function disableReloadButton() {
     reloadButton.classList.add('disabled')
 }
 
+function disableRunButton() {
+    const runButton = document.getElementById('run')
+    runButton.value = 'Reloading...'
+    runButton.disabled = true
+    runButton.classList.add('disabled')
+}
+
 function enableReloadButton() {
     const reloadButton = document.getElementById('reload')
     reloadButton.value = 'Reload'
@@ -1057,11 +1067,19 @@ function enableReloadButton() {
     reloadButton.classList.remove('disabled')
 }
 
+function enableRunButton() {
+    const runButton = document.getElementById('run')
+    runButton.value = 'Ok'
+    runButton.disabled = false
+    runButton.classList.remove('disabled')
+}
+
 function reloadAll() {
     updateParams();
     drawAll();
     saveState();
-    disableReloadButton()
+    disableReloadButton();
+    disableRunButton();
 }
 
 document.getElementById('params').onsubmit = function(event) {

rust/skim/Cargo.lock

@@ -14,13 +14,19 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "0.7.20"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
+checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
 dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"
@@ -32,9 +38,9 @@ dependencies = [
 
 [[package]]
 name = "arrayvec"
-version = "0.7.2"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
+checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
 
 [[package]]
 name = "atty"
@@ -42,7 +48,7 @@ version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.1.19",
  "libc",
  "winapi",
 ]
@@ -67,15 +73,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bumpalo"
-version = "3.11.1"
+version = "3.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
+checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
 
 [[package]]
 name = "cc"
-version = "1.0.77"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
 
 [[package]]
 name = "cfg-if"
@@ -85,13 +91,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.23"
+version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
+checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
 dependencies = [
+ "android-tzdata",
  "iana-time-zone",
  "js-sys",
- "num-integer",
  "num-traits",
  "time 0.1.45",
  "wasm-bindgen",
@@ -100,9 +106,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "3.2.23"
+version = "3.2.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
+checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
 dependencies = [
  "atty",
  "bitflags",
@@ -135,9 +141,9 @@ dependencies = [
 
 [[package]]
 name = "core-foundation-sys"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
 
 [[package]]
 name = "crossbeam"
@@ -155,9 +161,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.6"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
+checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
 dependencies = [
  "cfg-if",
  "crossbeam-utils",
@@ -165,9 +171,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-deque"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
+checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
 dependencies = [
  "cfg-if",
  "crossbeam-epoch",
@@ -176,14 +182,14 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.13"
+version = "0.9.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
+checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
 dependencies = [
  "autocfg",
  "cfg-if",
  "crossbeam-utils",
- "memoffset 0.7.1",
+ "memoffset 0.9.0",
  "scopeguard",
 ]
 
@@ -199,18 +205,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.14"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
 dependencies = [
  "cfg-if",
 ]
 
 [[package]]
 name = "cxx"
-version = "1.0.83"
+version = "1.0.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf"
+checksum = "e88abab2f5abbe4c56e8f1fb431b784d710b709888f35755a160e62e33fe38e8"
 dependencies = [
  "cc",
  "cxxbridge-flags",
@@ -220,9 +226,9 @@ dependencies = [
 
 [[package]]
 name = "cxx-build"
-version = "1.0.83"
+version = "1.0.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39"
+checksum = "5c0c11acd0e63bae27dcd2afced407063312771212b7a823b4fd72d633be30fb"
 dependencies = [
  "cc",
  "codespan-reporting",
@@ -230,31 +236,31 @@ dependencies = [
  "proc-macro2",
  "quote",
  "scratch",
- "syn",
+ "syn 2.0.23",
 ]
 
 [[package]]
 name = "cxxbridge-flags"
-version = "1.0.83"
+version = "1.0.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12"
+checksum = "8d3816ed957c008ccd4728485511e3d9aaf7db419aa321e3d2c5a2f3411e36c8"
 
 [[package]]
 name = "cxxbridge-macro"
-version = "1.0.83"
+version = "1.0.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6"
+checksum = "a26acccf6f445af85ea056362561a24ef56cdc15fcc685f03aec50b9c702cb6d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
 ]
 
 [[package]]
 name = "darling"
-version = "0.14.2"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa"
+checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -262,27 +268,27 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.14.2"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f"
+checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
  "strsim",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
 name = "darling_macro"
-version = "0.14.2"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e"
+checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
 dependencies = [
  "darling_core",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -313,7 +319,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -323,7 +329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68"
 dependencies = [
  "derive_builder_core",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -349,9 +355,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.8.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
+checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
 
 [[package]]
 name = "env_logger"
@@ -383,9 +389,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.8"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
+checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
 dependencies = [
  "cfg-if",
  "libc",
@@ -407,6 +413,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "hermit-abi"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+
 [[package]]
 name = "humantime"
 version = "2.1.0"
@@ -415,26 +427,25 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.53"
+version = "0.1.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
+checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
  "iana-time-zone-haiku",
  "js-sys",
  "wasm-bindgen",
- "winapi",
+ "windows",
 ]
 
 [[package]]
 name = "iana-time-zone-haiku"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
 dependencies = [
- "cxx",
- "cxx-build",
+ "cc",
 ]
 
 [[package]]
@@ -445,9 +456,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "indexmap"
-version = "1.9.2"
+version = "1.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
 dependencies = [
  "autocfg",
  "hashbrown",
@@ -455,9 +466,9 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.60"
+version = "0.3.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
+checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -470,27 +481,24 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.138"
+version = "0.2.147"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
 
 [[package]]
 name = "link-cplusplus"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369"
+checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "log"
-version = "0.4.17"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
-dependencies = [
- "cfg-if",
-]
+checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
 
 [[package]]
 name = "memchr"
@@ -509,9 +517,9 @@ dependencies = [
 
 [[package]]
 name = "memoffset"
-version = "0.7.1"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
 dependencies = [
  "autocfg",
 ]
@@ -541,16 +549,6 @@ dependencies = [
  "pin-utils",
 ]
 
-[[package]]
-name = "num-integer"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
-dependencies = [
- "autocfg",
- "num-traits",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.15"
@@ -562,25 +560,25 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.14.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.1",
  "libc",
 ]
 
 [[package]]
 name = "once_cell"
-version = "1.16.0"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
+checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "os_str_bytes"
-version = "6.4.1"
+version = "6.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
+checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac"
 
 [[package]]
 name = "pin-utils"
@@ -590,27 +588,27 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.47"
+version = "1.0.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
+checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.21"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
 dependencies = [
  "proc-macro2",
 ]
 
 [[package]]
 name = "rayon"
-version = "1.6.1"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
+checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
 dependencies = [
  "either",
  "rayon-core",
@@ -618,9 +616,9 @@ dependencies = [
 
 [[package]]
 name = "rayon-core"
-version = "1.10.1"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
+checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
 dependencies = [
  "crossbeam-channel",
  "crossbeam-deque",
@@ -650,9 +648,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.7.0"
+version = "1.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
+checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -661,15 +659,15 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.28"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
+checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
 
 [[package]]
 name = "rustversion"
-version = "1.0.9"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
+checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06"
 
 [[package]]
 name = "scopeguard"
@@ -679,15 +677,15 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "scratch"
-version = "1.0.2"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898"
+checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1"
 
 [[package]]
 name = "serde"
-version = "1.0.149"
+version = "1.0.164"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055"
+checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d"
 
 [[package]]
 name = "shlex"
@@ -697,9 +695,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
 
 [[package]]
 name = "skim"
-version = "0.10.2"
+version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cebed5f897cd6c0d80fbe30adb36c0abf7400e93043a63ae56458495642b3485"
+checksum = "e5d28de0a6cb2cdd83a076f1de9d965b973ae08b244df1aa70b432946dda0f32"
 dependencies = [
  "atty",
  "beef",
@@ -717,7 +715,7 @@ dependencies = [
  "rayon",
  "regex",
  "shlex",
- "time 0.3.17",
+ "time 0.3.22",
  "timer",
  "tuikit",
  "unicode-width",
@@ -732,9 +730,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
-version = "1.0.105"
+version = "1.0.109"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -754,9 +763,9 @@ dependencies = [
 
 [[package]]
 name = "termcolor"
-version = "1.1.3"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
 dependencies = [
  "winapi-util",
 ]
@@ -769,30 +778,31 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
 
 [[package]]
 name = "thiserror"
-version = "1.0.37"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
+checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.37"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
+checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
 ]
 
 [[package]]
 name = "thread_local"
-version = "1.1.4"
+version = "1.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
+checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
 dependencies = [
+ "cfg-if",
  "once_cell",
 ]
 
@@ -809,9 +819,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.17"
+version = "0.3.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
+checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd"
 dependencies = [
  "serde",
  "time-core",
@@ -819,9 +829,9 @@ dependencies = [
 
 [[package]]
 name = "time-core"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
+checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
 
 [[package]]
 name = "timer"
@@ -848,9 +858,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.5"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
+checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0"
 
 [[package]]
 name = "unicode-width"
@@ -860,15 +870,15 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
 
 [[package]]
 name = "utf8parse"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 
 [[package]]
 name = "vte"
-version = "0.11.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1aae21c12ad2ec2d168c236f369c38ff332bc1134f7246350dca641437365045"
+checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197"
 dependencies = [
  "arrayvec",
  "utf8parse",
@@ -899,9 +909,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.83"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
+checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -909,24 +919,24 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.83"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
+checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.83"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
+checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -934,22 +944,22 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.83"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
+checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.83"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
+checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
 
 [[package]]
 name = "winapi"
@@ -981,3 +991,69 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"

src/Access/Common/AccessRightsElement.cpp

@@ -155,7 +155,7 @@ namespace
 
 
 AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_)
-    : access_flags(access_flags_), database(database_), any_database(false)
+    : access_flags(access_flags_), database(database_), parameter(database_), any_database(false), any_parameter(false)
 {
 }
 

src/Access/Common/AccessType.h

@@ -70,7 +70,7 @@ enum class AccessType
     M(ALTER_FREEZE_PARTITION, "FREEZE PARTITION, UNFREEZE", TABLE, ALTER_TABLE) \
     \
     M(ALTER_DATABASE_SETTINGS, "ALTER DATABASE SETTING, ALTER MODIFY DATABASE SETTING, MODIFY DATABASE SETTING", DATABASE, ALTER_DATABASE) /* allows to execute ALTER MODIFY SETTING */\
-    M(ALTER_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) /* allows to execute ALTER NAMED COLLECTION */\
+    M(ALTER_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) /* allows to execute ALTER NAMED COLLECTION */\
     \
     M(ALTER_TABLE, "", GROUP, ALTER) \
     M(ALTER_DATABASE, "", GROUP, ALTER) \
@@ -92,7 +92,7 @@ enum class AccessType
     M(CREATE_ARBITRARY_TEMPORARY_TABLE, "", GLOBAL, CREATE)  /* allows to create  and manipulate temporary tables
                                                                 with arbitrary table engine */\
     M(CREATE_FUNCTION, "", GLOBAL, CREATE) /* allows to execute CREATE FUNCTION */ \
-    M(CREATE_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) /* allows to execute CREATE NAMED COLLECTION */ \
+    M(CREATE_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) /* allows to execute CREATE NAMED COLLECTION */ \
     M(CREATE, "", GROUP, ALL) /* allows to execute {CREATE|ATTACH} */ \
     \
     M(DROP_DATABASE, "", DATABASE, DROP) /* allows to execute {DROP|DETACH} DATABASE */\
@@ -101,7 +101,7 @@ enum class AccessType
                                     implicitly enabled by the grant DROP_TABLE */\
     M(DROP_DICTIONARY, "", DICTIONARY, DROP) /* allows to execute {DROP|DETACH} DICTIONARY */\
     M(DROP_FUNCTION, "", GLOBAL, DROP) /* allows to execute DROP FUNCTION */\
-    M(DROP_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) /* allows to execute DROP NAMED COLLECTION */\
+    M(DROP_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) /* allows to execute DROP NAMED COLLECTION */\
     M(DROP, "", GROUP, ALL) /* allows to execute {DROP|DETACH} */\
     \
     M(UNDROP_TABLE, "", TABLE, ALL) /* allows to execute {UNDROP} TABLE */\
@@ -140,9 +140,10 @@ enum class AccessType
     M(SHOW_SETTINGS_PROFILES, "SHOW PROFILES, SHOW CREATE SETTINGS PROFILE, SHOW CREATE PROFILE", GLOBAL, SHOW_ACCESS) \
     M(SHOW_ACCESS, "", GROUP, ACCESS_MANAGEMENT) \
     M(ACCESS_MANAGEMENT, "", GROUP, ALL) \
-    M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) \
-    M(SHOW_NAMED_COLLECTIONS_SECRETS, "SHOW NAMED COLLECTIONS SECRETS", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) \
-    M(NAMED_COLLECTION_CONTROL, "", NAMED_COLLECTION, ALL) \
+    M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) \
+    M(SHOW_NAMED_COLLECTIONS_SECRETS, "SHOW NAMED COLLECTIONS SECRETS", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) \
+    M(NAMED_COLLECTION, "NAMED COLLECTION USAGE, USE NAMED COLLECTION", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) \
+    M(NAMED_COLLECTION_ADMIN, "NAMED COLLECTION CONTROL", NAMED_COLLECTION, ALL) \
     \
     M(SYSTEM_SHUTDOWN, "SYSTEM KILL, SHUTDOWN", GLOBAL, SYSTEM) \
     M(SYSTEM_DROP_DNS_CACHE, "SYSTEM DROP DNS, DROP DNS CACHE, DROP DNS", GLOBAL, SYSTEM_DROP_CACHE) \

src/Access/ContextAccess.cpp

@@ -228,6 +228,12 @@ ContextAccess::ContextAccess(const AccessControl & access_control_, const Params
 }
 
 
+ContextAccess::ContextAccess(FullAccess)
+    : is_full_access(true), access(std::make_shared<AccessRights>(AccessRights::getFullAccess())), access_with_implicit(access)
+{
+}
+
+
 ContextAccess::~ContextAccess()
 {
     enabled_settings.reset();
@@ -413,14 +419,8 @@ std::optional<QuotaUsage> ContextAccess::getQuotaUsage() const
 
 std::shared_ptr<const ContextAccess> ContextAccess::getFullAccess()
 {
-    static const std::shared_ptr<const ContextAccess> res = []
-    {
-        auto full_access = std::make_shared<ContextAccess>();
-        full_access->is_full_access = true;
-        full_access->access = std::make_shared<AccessRights>(AccessRights::getFullAccess());
-        full_access->access_with_implicit = full_access->access;
-        return full_access;
-    }();
+    static const std::shared_ptr<const ContextAccess> res =
+        [] { return std::shared_ptr<ContextAccess>(new ContextAccess{kFullAccess}); }();
     return res;
 }
 

src/Access/ContextAccess.h

@@ -69,7 +69,6 @@ public:
     using Params = ContextAccessParams;
     const Params & getParams() const { return params; }
 
-    ContextAccess() { } /// NOLINT
     ContextAccess(const AccessControl & access_control_, const Params & params_);
 
     /// Returns the current user. Throws if user is nullptr.
@@ -171,10 +170,17 @@ public:
 private:
     friend class AccessControl;
 
+    struct FullAccess {};
+    static const FullAccess kFullAccess;
+
+    /// Makes an instance of ContextAccess which provides full access to everything
+    /// without any limitations. This is used for the global context.
+    explicit ContextAccess(FullAccess);
+
     void initialize();
-    void setUser(const UserPtr & user_) const;
-    void setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> & roles_info_) const;
-    void calculateAccessRights() const;
+    void setUser(const UserPtr & user_) const TSA_REQUIRES(mutex);
+    void setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> & roles_info_) const TSA_REQUIRES(mutex);
+    void calculateAccessRights() const TSA_REQUIRES(mutex);
 
     template <bool throw_if_denied, bool grant_option>
     bool checkAccessImpl(const AccessFlags & flags) const;
@@ -217,20 +223,23 @@ private:
 
     const AccessControl * access_control = nullptr;
     const Params params;
-    bool is_full_access = false;
-    mutable Poco::Logger * trace_log = nullptr;
-    mutable UserPtr user;
-    mutable String user_name;
-    mutable bool user_was_dropped = false;
-    mutable scope_guard subscription_for_user_change;
-    mutable std::shared_ptr<const EnabledRoles> enabled_roles;
-    mutable scope_guard subscription_for_roles_changes;
-    mutable std::shared_ptr<const EnabledRolesInfo> roles_info;
-    mutable std::shared_ptr<const AccessRights> access;
-    mutable std::shared_ptr<const AccessRights> access_with_implicit;
-    mutable std::shared_ptr<const EnabledRowPolicies> enabled_row_policies;
-    mutable std::shared_ptr<const EnabledQuota> enabled_quota;
-    mutable std::shared_ptr<const EnabledSettings> enabled_settings;
+    const bool is_full_access = false;
+
+    mutable std::atomic<bool> user_was_dropped = false;
+    mutable std::atomic<Poco::Logger *> trace_log = nullptr;
+
+    mutable UserPtr user TSA_GUARDED_BY(mutex);
+    mutable String user_name TSA_GUARDED_BY(mutex);
+    mutable scope_guard subscription_for_user_change TSA_GUARDED_BY(mutex);
+    mutable std::shared_ptr<const EnabledRoles> enabled_roles TSA_GUARDED_BY(mutex);
+    mutable scope_guard subscription_for_roles_changes TSA_GUARDED_BY(mutex);
+    mutable std::shared_ptr<const EnabledRolesInfo> roles_info TSA_GUARDED_BY(mutex);
+    mutable std::shared_ptr<const AccessRights> access TSA_GUARDED_BY(mutex);
+    mutable std::shared_ptr<const AccessRights> access_with_implicit TSA_GUARDED_BY(mutex);
+    mutable std::shared_ptr<const EnabledRowPolicies> enabled_row_policies TSA_GUARDED_BY(mutex);
+    mutable std::shared_ptr<const EnabledQuota> enabled_quota TSA_GUARDED_BY(mutex);
+    mutable std::shared_ptr<const EnabledSettings> enabled_settings TSA_GUARDED_BY(mutex);
+
     mutable std::mutex mutex;
 };
 

src/Access/UsersConfigAccessStorage.cpp

@@ -328,7 +328,7 @@ namespace
 
             if (!named_collection_control)
             {
-                user->access.revoke(AccessType::NAMED_COLLECTION_CONTROL);
+                user->access.revoke(AccessType::NAMED_COLLECTION_ADMIN);
             }
 
             if (!show_named_collections_secrets)

src/Access/tests/gtest_access_rights_ops.cpp

@@ -53,7 +53,7 @@ TEST(AccessRights, Union)
               "SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
               "SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
               "SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
-              "SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*, GRANT NAMED COLLECTION CONTROL ON db1");
+              "SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*, GRANT NAMED COLLECTION ADMIN ON db1");
 }
 
 

src/Backups/tests/gtest_backup_entries.cpp

@@ -24,7 +24,7 @@ protected:
         /// Make local disk.
         temp_dir = std::make_unique<Poco::TemporaryFile>();
         temp_dir->createDirectories();
-        local_disk = std::make_shared<DiskLocal>("local_disk", temp_dir->path() + "/", 0);
+        local_disk = std::make_shared<DiskLocal>("local_disk", temp_dir->path() + "/");
 
         /// Make encrypted disk.
         auto settings = std::make_unique<DiskEncryptedSettings>();
@@ -38,7 +38,7 @@ protected:
         settings->current_key = key;
         settings->current_key_fingerprint = fingerprint;
 
-        encrypted_disk = std::make_shared<DiskEncrypted>("encrypted_disk", std::move(settings), true);
+        encrypted_disk = std::make_shared<DiskEncrypted>("encrypted_disk", std::move(settings));
     }
 
     void TearDown() override

src/Client/ClientBase.cpp

@@ -2297,7 +2297,9 @@ void ClientBase::runInteractive()
         catch (const ErrnoException & e)
         {
             if (e.getErrno() != EEXIST)
-                throw;
+            {
+                std::cerr << getCurrentExceptionMessage(false) << '\n';
+            }
         }
     }
 

src/Client/ConnectionParameters.cpp

@@ -60,7 +60,15 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
     quota_key = config.getString("quota_key", "");
 
     /// By default compression is disabled if address looks like localhost.
-    compression = config.getBool("compression", !isLocalAddress(DNSResolver::instance().resolveHost(host)))
+
+    /// Avoid DNS request if the host is "localhost".
+    /// If ClickHouse is run under QEMU-user with a binary for a different architecture,
+    /// and there are all listed startup dependency shared libraries available, but not the runtime dependencies of glibc,
+    /// the glibc cannot open "plugins" for DNS resolving, and the DNS resolution does not work.
+    /// At the same time, I want clickhouse-local to always work, regardless.
+    /// TODO: get rid of glibc, or replace getaddrinfo to c-ares.
+
+    compression = config.getBool("compression", host != "localhost" && !isLocalAddress(DNSResolver::instance().resolveHost(host)))
                   ? Protocol::Compression::Enable : Protocol::Compression::Disable;
 
     timeouts = ConnectionTimeouts(

src/Client/Suggest.cpp

@@ -101,9 +101,8 @@ static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggesti
         add_column("name", "columns", true, suggestion_limit);
     }
 
-    /// FIXME: Forbid this query using new analyzer because of bug https://github.com/ClickHouse/ClickHouse/issues/50669
-    /// We should remove this restriction after resolving this bug.
-    query = "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + query + ") WHERE notEmpty(res) SETTINGS allow_experimental_analyzer=0";
+    /// FIXME: This query does not work with the new analyzer because of bug https://github.com/ClickHouse/ClickHouse/issues/50669
+    query = "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + query + ") WHERE notEmpty(res)";
     return query;
 }
 

src/Columns/ColumnVector.h

@@ -107,8 +107,8 @@ struct FloatCompareHelper
     }
 };
 
-template <class U> struct CompareHelper<Float32, U> : public FloatCompareHelper<Float32> {};
-template <class U> struct CompareHelper<Float64, U> : public FloatCompareHelper<Float64> {};
+template <typename U> struct CompareHelper<Float32, U> : public FloatCompareHelper<Float32> {};
+template <typename U> struct CompareHelper<Float64, U> : public FloatCompareHelper<Float64> {};
 
 
 /** A template for columns that use a simple array to store.

src/Common/CurrentMetrics.cpp

@@ -93,8 +93,8 @@
     M(ThreadPoolFSReaderThreadsActive, "Number of threads in the thread pool for local_filesystem_read_method=threadpool running a task.") \
     M(BackupsIOThreads, "Number of threads in the BackupsIO thread pool.") \
     M(BackupsIOThreadsActive, "Number of threads in the BackupsIO thread pool running a task.") \
-    M(DiskObjectStorageAsyncThreads, "Number of threads in the async thread pool for DiskObjectStorage.") \
-    M(DiskObjectStorageAsyncThreadsActive, "Number of threads in the async thread pool for DiskObjectStorage running a task.") \
+    M(DiskObjectStorageAsyncThreads, "Obsolete metric, shows nothing.") \
+    M(DiskObjectStorageAsyncThreadsActive, "Obsolete metric, shows nothing.") \
     M(StorageHiveThreads, "Number of threads in the StorageHive thread pool.") \
     M(StorageHiveThreadsActive, "Number of threads in the StorageHive thread pool running a task.") \
     M(TablesLoaderThreads, "Number of threads in the tables loader thread pool.") \
@@ -141,6 +141,8 @@
     M(MergeTreeOutdatedPartsLoaderThreadsActive, "Number of active threads in the threadpool for loading Outdated data parts.") \
     M(MergeTreePartsCleanerThreads, "Number of threads in the MergeTree parts cleaner thread pool.") \
     M(MergeTreePartsCleanerThreadsActive, "Number of threads in the MergeTree parts cleaner thread pool running a task.") \
+    M(IDiskCopierThreads, "Number of threads for copying data between disks of different types.") \
+    M(IDiskCopierThreadsActive, "Number of threads for copying data between disks of different types running a task.") \
     M(SystemReplicasThreads, "Number of threads in the system.replicas thread pool.") \
     M(SystemReplicasThreadsActive, "Number of threads in the system.replicas thread pool running a task.") \
     M(RestartReplicaThreads, "Number of threads in the RESTART REPLICA thread pool.") \
@@ -188,6 +190,7 @@
     M(CacheDetachedFileSegments, "Number of existing detached cache file segments") \
     M(FilesystemCacheSize, "Filesystem cache size in bytes") \
     M(FilesystemCacheElements, "Filesystem cache elements (file segments)") \
+    M(FilesystemCacheDownloadQueueElements, "Filesystem cache elements in download queue") \
     M(AsyncInsertCacheSize, "Number of async insert hash id in cache") \
     M(S3Requests, "S3 requests") \
     M(KeeperAliveConnections, "Number of alive connections") \

src/Common/Exception.cpp

@@ -418,6 +418,18 @@ PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, b
                 << " (version " << VERSION_STRING << VERSION_OFFICIAL << ")";
         }
         catch (...) {}
+
+// #ifdef ABORT_ON_LOGICAL_ERROR
+//         try
+//         {
+//             throw;
+//         }
+//         catch (const std::logic_error &)
+//         {
+//             abortOnFailedAssertion(stream.str());
+//         }
+//         catch (...) {}
+// #endif
     }
     catch (...)
     {

src/Common/OptimizedRegularExpression.cpp

@@ -540,7 +540,7 @@ bool OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject, si
             }
         }
 
-        return re2->Match(StringPieceType(subject, subject_size), 0, subject_size, RegexType::UNANCHORED, nullptr, 0);
+        return re2->Match({subject, subject_size}, 0, subject_size, RegexType::UNANCHORED, nullptr, 0);
     }
 }
 
@@ -585,9 +585,9 @@ bool OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject, si
                 return false;
         }
 
-        StringPieceType piece;
+        std::string_view piece;
 
-        if (!RegexType::PartialMatch(StringPieceType(subject, subject_size), *re2, &piece))
+        if (!RegexType::PartialMatch({subject, subject_size}, *re2, &piece))
             return false;
         else
         {
@@ -652,10 +652,10 @@ unsigned OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject
                 return 0;
         }
 
-        DB::PODArrayWithStackMemory<StringPieceType, 128> pieces(limit);
+        DB::PODArrayWithStackMemory<std::string_view, 128> pieces(limit);
 
         if (!re2->Match(
-            StringPieceType(subject, subject_size),
+            {subject, subject_size},
             0,
             subject_size,
             RegexType::UNANCHORED,

src/Common/OptimizedRegularExpression.h

@@ -52,7 +52,6 @@ public:
     using MatchVec = std::vector<Match>;
 
     using RegexType = std::conditional_t<thread_safe, re2::RE2, re2_st::RE2>;
-    using StringPieceType = std::conditional_t<thread_safe, re2::StringPiece, re2_st::StringPiece>;
 
     OptimizedRegularExpressionImpl(const std::string & regexp_, int options = 0); /// NOLINT
     /// StringSearcher store pointers to required_substring, it must be updated on move.

src/Common/SensitiveDataMasker.cpp

@@ -5,7 +5,6 @@
 #include <atomic>
 
 #include <re2/re2.h>
-#include <re2/stringpiece.h>
 
 #include <Poco/Util/AbstractConfiguration.h>
 
@@ -44,7 +43,7 @@ private:
     const std::string regexp_string;
 
     const RE2 regexp;
-    const re2::StringPiece replacement;
+    const std::string_view replacement;
 
 #ifndef NDEBUG
     mutable std::atomic<std::uint64_t> matches_count = 0;

src/Common/ThreadStatus.cpp

@@ -67,8 +67,8 @@ ThreadGroup::ThreadGroup()
     : master_thread_id(CurrentThread::get().thread_id)
 {}
 
-ThreadStatus::ThreadStatus()
-    : thread_id{getThreadId()}
+ThreadStatus::ThreadStatus(bool check_current_thread_on_destruction_)
+    : thread_id{getThreadId()}, check_current_thread_on_destruction(check_current_thread_on_destruction_)
 {
     last_rusage = std::make_unique<RUsageCounters>();
 
@@ -201,8 +201,11 @@ ThreadStatus::~ThreadStatus()
 
     /// Only change current_thread if it's currently being used by this ThreadStatus
     /// For example, PushingToViews chain creates and deletes ThreadStatus instances while running in the main query thread
-    if (current_thread == this)
+    if (check_current_thread_on_destruction)
+    {
+        assert(current_thread == this);
         current_thread = nullptr;
+    }
 }
 
 void ThreadStatus::updatePerformanceCounters()

src/Common/ThreadStatus.h

@@ -224,8 +224,10 @@ private:
 
     Poco::Logger * log = nullptr;
 
+    bool check_current_thread_on_destruction;
+
 public:
-    ThreadStatus();
+    explicit ThreadStatus(bool check_current_thread_on_destruction_ = true);
     ~ThreadStatus();
 
     ThreadGroupPtr getThreadGroup() const;

src/Common/checkStackSize.cpp

@@ -27,7 +27,7 @@ static thread_local size_t max_stack_size = 0;
  * @param out_address - if not nullptr, here the address of the stack will be written.
  * @return stack size
  */
-size_t getStackSize(void ** out_address)
+static size_t getStackSize(void ** out_address)
 {
     using namespace DB;
 
@@ -54,7 +54,15 @@ size_t getStackSize(void ** out_address)
         throwFromErrno("Cannot pthread_attr_get_np", ErrorCodes::CANNOT_PTHREAD_ATTR);
 #   else
     if (0 != pthread_getattr_np(pthread_self(), &attr))
-        throwFromErrno("Cannot pthread_getattr_np", ErrorCodes::CANNOT_PTHREAD_ATTR);
+    {
+        if (errno == ENOENT)
+        {
+            /// Most likely procfs is not mounted.
+            return 0;
+        }
+        else
+            throwFromErrno("Cannot pthread_getattr_np", ErrorCodes::CANNOT_PTHREAD_ATTR);
+    }
 #   endif
 
     SCOPE_EXIT({ pthread_attr_destroy(&attr); });
@@ -83,6 +91,10 @@ __attribute__((__weak__)) void checkStackSize()
     if (!stack_address)
         max_stack_size = getStackSize(&stack_address);
 
+    /// The check is impossible.
+    if (!max_stack_size)
+        return;
+
     const void * frame_address = __builtin_frame_address(0);
     uintptr_t int_frame_address = reinterpret_cast<uintptr_t>(frame_address);
     uintptr_t int_stack_address = reinterpret_cast<uintptr_t>(stack_address);

src/Common/getHashOfLoadedBinary.cpp

@@ -37,7 +37,7 @@ SipHash getHashOfLoadedBinary()
 std::string getHashOfLoadedBinaryHex()
 {
     SipHash hash = getHashOfLoadedBinary();
-    std::array<UInt64, 2> checksum;
+    UInt128 checksum;
     hash.get128(checksum);
     return getHexUIntUppercase(checksum);
 }

src/Common/parseGlobs.cpp

@@ -3,7 +3,6 @@
 #include <IO/ReadBufferFromString.h>
 #include <IO/Operators.h>
 #include <re2/re2.h>
-#include <re2/stringpiece.h>
 #include <algorithm>
 #include <sstream>
 #include <iomanip>
@@ -33,14 +32,14 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob
     std::string escaped_with_globs = buf_for_escaping.str();
 
     static const re2::RE2 enum_or_range(R"({([\d]+\.\.[\d]+|[^{}*,]+,[^{}*]*[^{}*,])})");    /// regexp for {expr1,expr2,expr3} or {M..N}, where M and N - non-negative integers, expr's should be without "{", "}", "*" and ","
-    re2::StringPiece input(escaped_with_globs);
-    re2::StringPiece matched;
+    std::string_view input(escaped_with_globs);
+    std::string_view matched;
     std::ostringstream oss_for_replacing;       // STYLE_CHECK_ALLOW_STD_STRING_STREAM
     oss_for_replacing.exceptions(std::ios::failbit);
     size_t current_index = 0;
     while (RE2::FindAndConsume(&input, enum_or_range, &matched))
     {
-        std::string buffer{matched};
+        std::string buffer(matched);
         oss_for_replacing << escaped_with_globs.substr(current_index, matched.data() - escaped_with_globs.data() - current_index - 1) << '(';
 
         if (buffer.find(',') == std::string::npos)

src/Compression/CompressedReadBufferBase.cpp

@@ -49,8 +49,8 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c
 
     /// TODO mess up of endianness in error message.
     message << "Checksum doesn't match: corrupted data."
-        " Reference: " + getHexUIntLowercase(expected_checksum.high64) + getHexUIntLowercase(expected_checksum.low64)
-        + ". Actual: " + getHexUIntLowercase(calculated_checksum.high64) + getHexUIntLowercase(calculated_checksum.low64)
+        " Reference: " + getHexUIntLowercase(expected_checksum)
+        + ". Actual: " + getHexUIntLowercase(calculated_checksum)
         + ". Size of compressed block: " + toString(size);
 
     const char * message_hardware_failure = "This is most likely due to hardware failure. "

src/Compression/CompressionCodecLZ4.cpp

@@ -42,7 +42,6 @@ private:
     UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
 
     mutable LZ4::PerformanceStatistics lz4_stat;
-    ASTPtr codec_desc;
 };
 
 

src/Coordination/KeeperContext.cpp

@@ -220,7 +220,7 @@ KeeperContext::Storage KeeperContext::getLogsPathFromConfig(const Poco::Util::Ab
         if (!fs::exists(path))
             fs::create_directories(path);
 
-        return std::make_shared<DiskLocal>("LocalLogDisk", path, 0);
+        return std::make_shared<DiskLocal>("LocalLogDisk", path);
     };
 
     /// the most specialized path
@@ -246,7 +246,7 @@ KeeperContext::Storage KeeperContext::getSnapshotsPathFromConfig(const Poco::Uti
         if (!fs::exists(path))
             fs::create_directories(path);
 
-        return std::make_shared<DiskLocal>("LocalSnapshotDisk", path, 0);
+        return std::make_shared<DiskLocal>("LocalSnapshotDisk", path);
     };
 
     /// the most specialized path
@@ -272,7 +272,7 @@ KeeperContext::Storage KeeperContext::getStatePathFromConfig(const Poco::Util::A
         if (!fs::exists(path))
             fs::create_directories(path);
 
-        return std::make_shared<DiskLocal>("LocalStateFileDisk", path, 0);
+        return std::make_shared<DiskLocal>("LocalStateFileDisk", path);
     };
 
     if (config.has("keeper_server.state_storage_disk"))

src/Coordination/tests/gtest_coordination.cpp

@@ -71,16 +71,16 @@ protected:
     DB::KeeperContextPtr keeper_context = std::make_shared<DB::KeeperContext>(true);
     Poco::Logger * log{&Poco::Logger::get("CoordinationTest")};
 
-    void setLogDirectory(const std::string & path) { keeper_context->setLogDisk(std::make_shared<DB::DiskLocal>("LogDisk", path, 0)); }
+    void setLogDirectory(const std::string & path) { keeper_context->setLogDisk(std::make_shared<DB::DiskLocal>("LogDisk", path)); }
 
     void setSnapshotDirectory(const std::string & path)
     {
-        keeper_context->setSnapshotDisk(std::make_shared<DB::DiskLocal>("SnapshotDisk", path, 0));
+        keeper_context->setSnapshotDisk(std::make_shared<DB::DiskLocal>("SnapshotDisk", path));
     }
 
     void setStateFileDirectory(const std::string & path)
     {
-        keeper_context->setStateFileDisk(std::make_shared<DB::DiskLocal>("StateFile", path, 0));
+        keeper_context->setStateFileDisk(std::make_shared<DB::DiskLocal>("StateFile", path));
     }
 };
 
@@ -1503,9 +1503,9 @@ void testLogAndStateMachine(
     using namespace DB;
 
     ChangelogDirTest snapshots("./snapshots");
-    keeper_context->setSnapshotDisk(std::make_shared<DiskLocal>("SnapshotDisk", "./snapshots", 0));
+    keeper_context->setSnapshotDisk(std::make_shared<DiskLocal>("SnapshotDisk", "./snapshots"));
     ChangelogDirTest logs("./logs");
-    keeper_context->setLogDisk(std::make_shared<DiskLocal>("LogDisk", "./logs", 0));
+    keeper_context->setLogDisk(std::make_shared<DiskLocal>("LogDisk", "./logs"));
 
     ResponsesQueue queue(std::numeric_limits<size_t>::max());
     SnapshotsQueue snapshots_queue{1};

src/Core/Settings.h

@@ -517,6 +517,7 @@ class IColumn;
     M(Seconds, wait_for_window_view_fire_signal_timeout, 10, "Timeout for waiting for window view fire signal in event time processing", 0) \
     M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \
     \
+    M(DefaultTableEngine, default_temporary_table_engine, DefaultTableEngine::Memory, "Default table engine used when ENGINE is not set in CREATE TEMPORARY statement.",0) \
     M(DefaultTableEngine, default_table_engine, DefaultTableEngine::None, "Default table engine used when ENGINE is not set in CREATE statement.",0) \
     M(Bool, show_table_uuid_in_table_create_query_if_not_nil, false, "For tables in databases with Engine=Atomic show UUID of the table in its CREATE query.", 0) \
     M(Bool, database_atomic_wait_for_drop_and_detach_synchronously, false, "When executing DROP or DETACH TABLE in Atomic database, wait for table data to be finally dropped or detached.", 0) \

src/Core/SettingsQuirks.cpp

@@ -1,10 +1,11 @@
+#include <base/defines.h>
 #include <Core/SettingsQuirks.h>
 #include <Core/Settings.h>
 #include <Poco/Environment.h>
 #include <Poco/Platform.h>
 #include <Common/VersionNumber.h>
 #include <Common/logger_useful.h>
-#include <cstdlib>
+
 
 namespace
 {
@@ -71,6 +72,12 @@ void applySettingsQuirks(Settings & settings, Poco::Logger * log)
         }
     }
 
+#if defined(THREAD_SANITIZER)
+    settings.use_hedged_requests.value = false;
+    if (log)
+        LOG_WARNING(log, "use_hedged_requests has been disabled for the build with Thread Sanitizer, because they are using fibers, leading to a failed assertion inside TSan");
+#endif
+
     if (!queryProfilerWorks())
     {
         if (settings.query_profiler_real_time_period_ns)

src/Databases/DatabaseFactory.cpp

@@ -3,6 +3,7 @@
 #include <filesystem>
 #include <Databases/DatabaseAtomic.h>
 #include <Databases/DatabaseDictionary.h>
+#include <Databases/DatabaseFilesystem.h>
 #include <Databases/DatabaseLazy.h>
 #include <Databases/DatabaseMemory.h>
 #include <Databases/DatabaseOrdinary.h>
@@ -47,6 +48,14 @@
 #include <Databases/SQLite/DatabaseSQLite.h>
 #endif
 
+#if USE_AWS_S3
+#include <Databases/DatabaseS3.h>
+#endif
+
+#if USE_HDFS
+#include <Databases/DatabaseHDFS.h>
+#endif
+
 namespace fs = std::filesystem;
 
 namespace DB
@@ -131,13 +140,13 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String
 
     static const std::unordered_set<std::string_view> database_engines{"Ordinary", "Atomic", "Memory",
         "Dictionary", "Lazy", "Replicated", "MySQL", "MaterializeMySQL", "MaterializedMySQL",
-        "PostgreSQL", "MaterializedPostgreSQL", "SQLite"};
+        "PostgreSQL", "MaterializedPostgreSQL", "SQLite", "Filesystem", "S3", "HDFS"};
 
     if (!database_engines.contains(engine_name))
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database engine name `{}` does not exist", engine_name);
 
     static const std::unordered_set<std::string_view> engines_with_arguments{"MySQL", "MaterializeMySQL", "MaterializedMySQL",
-        "Lazy", "Replicated", "PostgreSQL", "MaterializedPostgreSQL", "SQLite"};
+        "Lazy", "Replicated", "PostgreSQL", "MaterializedPostgreSQL", "SQLite", "Filesystem", "S3", "HDFS"};
 
     static const std::unordered_set<std::string_view> engines_with_table_overrides{"MaterializeMySQL", "MaterializedMySQL", "MaterializedPostgreSQL"};
     bool engine_may_have_arguments = engines_with_arguments.contains(engine_name);
@@ -432,6 +441,63 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String
     }
 #endif
 
+    else if (engine_name == "Filesystem")
+    {
+        const ASTFunction * engine = engine_define->engine;
+
+        /// If init_path is empty, then the current path will be used
+        std::string init_path;
+
+        if (engine->arguments && !engine->arguments->children.empty())
+        {
+            if (engine->arguments->children.size() != 1)
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Filesystem database requires at most 1 argument: filesystem_path");
+
+            const auto & arguments = engine->arguments->children;
+            init_path = safeGetLiteralValue<String>(arguments[0], engine_name);
+        }
+
+        return std::make_shared<DatabaseFilesystem>(database_name, init_path, context);
+    }
+
+#if USE_AWS_S3
+    else if (engine_name == "S3")
+    {
+        const ASTFunction * engine = engine_define->engine;
+
+        DatabaseS3::Configuration config;
+
+        if (engine->arguments && !engine->arguments->children.empty())
+        {
+            ASTs & engine_args = engine->arguments->children;
+            config = DatabaseS3::parseArguments(engine_args, context);
+        }
+
+        return std::make_shared<DatabaseS3>(database_name, config, context);
+    }
+#endif
+
+#if USE_HDFS
+    else if (engine_name == "HDFS")
+    {
+        const ASTFunction * engine = engine_define->engine;
+
+        /// If source_url is empty, then table name must contain full url
+        std::string source_url;
+
+        if (engine->arguments && !engine->arguments->children.empty())
+        {
+            if (engine->arguments->children.size() != 1)
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "HDFS database requires at most 1 argument: source_url");
+
+            const auto & arguments = engine->arguments->children;
+            source_url = safeGetLiteralValue<String>(arguments[0], engine_name);
+        }
+
+        return std::make_shared<DatabaseHDFS>(database_name, source_url, context);
+    }
+#endif
+
     throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Unknown database engine: {}", engine_name);
 }
 

src/Databases/DatabaseFilesystem.cpp

@@ -0,0 +1,245 @@
+#include <Databases/DatabaseFilesystem.h>
+
+#include <IO/Operators.h>
+#include <IO/WriteBufferFromString.h>
+#include <Interpreters/Context.h>
+#include <Parsers/ASTCreateQuery.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/ParserCreateQuery.h>
+#include <Parsers/parseQuery.h>
+#include <Storages/IStorage.h>
+#include <TableFunctions/TableFunctionFactory.h>
+#include <Common/filesystemHelpers.h>
+
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int UNKNOWN_TABLE;
+    extern const int PATH_ACCESS_DENIED;
+    extern const int BAD_ARGUMENTS;
+    extern const int FILE_DOESNT_EXIST;
+}
+
+DatabaseFilesystem::DatabaseFilesystem(const String & name_, const String & path_, ContextPtr context_)
+    : IDatabase(name_), WithContext(context_->getGlobalContext()), path(path_), log(&Poco::Logger::get("DatabaseFileSystem(" + name_ + ")"))
+{
+    bool is_local = context_->getApplicationType() == Context::ApplicationType::LOCAL;
+    fs::path user_files_path = is_local ? "" : fs::canonical(getContext()->getUserFilesPath());
+
+    if (fs::path(path).is_relative())
+    {
+        path = user_files_path / path;
+    }
+    else if (!is_local && !pathStartsWith(fs::path(path), user_files_path))
+    {
+        throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                        "Path must be inside user-files path: {}", user_files_path.string());
+    }
+
+    path = fs::absolute(path).lexically_normal();
+    if (!fs::exists(path))
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path does not exist: {}", path);
+}
+
+std::string DatabaseFilesystem::getTablePath(const std::string & table_name) const
+{
+    fs::path table_path = fs::path(path) / table_name;
+    return table_path.lexically_normal().string();
+}
+
+void DatabaseFilesystem::addTable(const std::string & table_name, StoragePtr table_storage) const
+{
+    std::lock_guard lock(mutex);
+    auto [_, inserted] = loaded_tables.emplace(table_name, table_storage);
+    if (!inserted)
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Table with name `{}` already exists in database `{}` (engine {})",
+            table_name, getDatabaseName(), getEngineName());
+}
+
+bool DatabaseFilesystem::checkTableFilePath(const std::string & table_path, ContextPtr context_, bool throw_on_error) const
+{
+    /// If run in Local mode, no need for path checking.
+    bool check_path = context_->getApplicationType() != Context::ApplicationType::LOCAL;
+    const auto & user_files_path = context_->getUserFilesPath();
+
+    /// Check access for file before checking its existence.
+    if (check_path && !fileOrSymlinkPathStartsWith(table_path, user_files_path))
+    {
+        if (throw_on_error)
+            throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File is not inside {}", user_files_path);
+        else
+            return false;
+    }
+
+    /// Check if the corresponding file exists.
+    if (!fs::exists(table_path))
+    {
+        if (throw_on_error)
+            throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "File does not exist: {}", table_path);
+        else
+            return false;
+    }
+
+    if (!fs::is_regular_file(table_path))
+    {
+        if (throw_on_error)
+            throw Exception(ErrorCodes::FILE_DOESNT_EXIST,
+                            "File is directory, but expected a file: {}", table_path);
+        else
+            return false;
+    }
+
+    return true;
+}
+
+StoragePtr DatabaseFilesystem::tryGetTableFromCache(const std::string & name) const
+{
+    StoragePtr table = nullptr;
+    {
+        std::lock_guard lock(mutex);
+        auto it = loaded_tables.find(name);
+        if (it != loaded_tables.end())
+            table = it->second;
+    }
+
+    /// Invalidate cache if file no longer exists.
+    if (table && !fs::exists(getTablePath(name)))
+    {
+        std::lock_guard lock(mutex);
+        loaded_tables.erase(name);
+        return nullptr;
+    }
+
+    return table;
+}
+
+bool DatabaseFilesystem::isTableExist(const String & name, ContextPtr context_) const
+{
+    if (tryGetTableFromCache(name))
+        return true;
+
+    return checkTableFilePath(getTablePath(name), context_, /* throw_on_error */false);
+}
+
+StoragePtr DatabaseFilesystem::getTableImpl(const String & name, ContextPtr context_) const
+{
+    /// Check if table exists in loaded tables map.
+    if (auto table = tryGetTableFromCache(name))
+        return table;
+
+    auto table_path = getTablePath(name);
+    checkTableFilePath(table_path, context_, /* throw_on_error */true);
+
+    /// If the file exists, create a new table using TableFunctionFile and return it.
+    auto args = makeASTFunction("file", std::make_shared<ASTLiteral>(table_path));
+
+    auto table_function = TableFunctionFactory::instance().get(args, context_);
+    if (!table_function)
+        return nullptr;
+
+    /// TableFunctionFile throws exceptions, if table cannot be created.
+    auto table_storage = table_function->execute(args, context_, name);
+    if (table_storage)
+        addTable(name, table_storage);
+
+    return table_storage;
+}
+
+StoragePtr DatabaseFilesystem::getTable(const String & name, ContextPtr context_) const
+{
+    /// getTableImpl can throw exceptions, do not catch them to show correct error to user.
+    if (auto storage = getTableImpl(name, context_))
+        return storage;
+
+    throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist",
+                    backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name));
+}
+
+StoragePtr DatabaseFilesystem::tryGetTable(const String & name, ContextPtr context_) const
+{
+    try
+    {
+        return getTableImpl(name, context_);
+    }
+    catch (const Exception & e)
+    {
+        /// Ignore exceptions thrown by TableFunctionFile, which indicate that there is no table
+        /// see tests/02722_database_filesystem.sh for more details.
+        if (e.code() == ErrorCodes::FILE_DOESNT_EXIST)
+        {
+            return nullptr;
+        }
+        throw;
+    }
+}
+
+bool DatabaseFilesystem::empty() const
+{
+    std::lock_guard lock(mutex);
+    return loaded_tables.empty();
+}
+
+ASTPtr DatabaseFilesystem::getCreateDatabaseQuery() const
+{
+    const auto & settings = getContext()->getSettingsRef();
+    const String query = fmt::format("CREATE DATABASE {} ENGINE = Filesystem('{}')", backQuoteIfNeed(getDatabaseName()), path);
+
+    ParserCreateQuery parser;
+    ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
+
+    if (const auto database_comment = getDatabaseComment(); !database_comment.empty())
+    {
+        auto & ast_create_query = ast->as<ASTCreateQuery &>();
+        ast_create_query.set(ast_create_query.comment, std::make_shared<ASTLiteral>(database_comment));
+    }
+
+    return ast;
+}
+
+void DatabaseFilesystem::shutdown()
+{
+    Tables tables_snapshot;
+    {
+        std::lock_guard lock(mutex);
+        tables_snapshot = loaded_tables;
+    }
+
+    for (const auto & kv : tables_snapshot)
+    {
+        auto table_id = kv.second->getStorageID();
+        kv.second->flushAndShutdown();
+    }
+
+    std::lock_guard lock(mutex);
+    loaded_tables.clear();
+}
+
+/**
+ * Returns an empty vector because the database is read-only and no tables can be backed up
+ */
+std::vector<std::pair<ASTPtr, StoragePtr>> DatabaseFilesystem::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const
+{
+    return {};
+}
+
+/**
+ *
+ * Returns an empty iterator because the database does not have its own tables
+ * But only caches them for quick access
+ */
+DatabaseTablesIteratorPtr DatabaseFilesystem::getTablesIterator(ContextPtr, const FilterByNameFunction &) const
+{
+    return std::make_unique<DatabaseTablesSnapshotIterator>(Tables{}, getDatabaseName());
+}
+
+}

src/Databases/DatabaseFilesystem.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <mutex>
+#include <Databases/IDatabase.h>
+#include <Parsers/IAST.h>
+#include <Storages/IStorage_fwd.h>
+#include <base/types.h>
+
+namespace DB
+{
+
+class Context;
+
+/**
+  * DatabaseFilesystem allows to interact with files stored on the local filesystem.
+  * Uses TableFunctionFile to implicitly load file when a user requests the table,
+  * and provides a read-only access to the data in the file.
+  * Tables are cached inside the database for quick access
+  *
+  * Used in clickhouse-local to access local files.
+  * For clickhouse-server requires allows to access file only from user_files directory.
+  */
+class DatabaseFilesystem : public IDatabase, protected WithContext
+{
+public:
+    DatabaseFilesystem(const String & name, const String & path, ContextPtr context);
+
+    String getEngineName() const override { return "Filesystem"; }
+
+    bool isTableExist(const String & name, ContextPtr context) const override;
+
+    StoragePtr getTable(const String & name, ContextPtr context) const override;
+
+    StoragePtr tryGetTable(const String & name, ContextPtr context) const override;
+
+    bool shouldBeEmptyOnDetach() const override { return false; } /// Contains only temporary tables.
+
+    bool empty() const override;
+
+    bool isReadOnly() const override { return true; }
+
+    ASTPtr getCreateDatabaseQuery() const override;
+
+    void shutdown() override;
+
+    std::vector<std::pair<ASTPtr, StoragePtr>> getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const override;
+
+    DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &) const override;
+
+protected:
+    StoragePtr getTableImpl(const String & name, ContextPtr context) const;
+
+    StoragePtr tryGetTableFromCache(const std::string & name) const;
+
+    std::string getTablePath(const std::string & table_name) const;
+
+    void addTable(const std::string & table_name, StoragePtr table_storage) const;
+
+    bool checkTableFilePath(const std::string & table_path, ContextPtr context_, bool throw_on_error) const;
+
+private:
+    String path;
+    mutable Tables loaded_tables TSA_GUARDED_BY(mutex);
+    Poco::Logger * log;
+};
+
+}

src/Databases/DatabaseHDFS.cpp

@@ -0,0 +1,234 @@
+#include "config.h"
+
+#if USE_HDFS
+
+#include <Databases/DatabaseHDFS.h>
+
+#include <Interpreters/Context.h>
+#include <Parsers/ASTCreateQuery.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/parseQuery.h>
+#include <Parsers/ParserCreateQuery.h>
+#include <Storages/HDFS/HDFSCommon.h>
+#include <Storages/IStorage.h>
+#include <TableFunctions/TableFunctionFactory.h>
+
+#include <Poco/URI.h>
+#include <re2/re2.h>
+
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int UNKNOWN_TABLE;
+    extern const int BAD_ARGUMENTS;
+    extern const int FILE_DOESNT_EXIST;
+    extern const int UNACCEPTABLE_URL;
+    extern const int ACCESS_DENIED;
+    extern const int DATABASE_ACCESS_DENIED;
+    extern const int HDFS_ERROR;
+    extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
+}
+
+static constexpr std::string_view HDFS_HOST_REGEXP = "^hdfs://[^/]*";
+
+
+DatabaseHDFS::DatabaseHDFS(const String & name_, const String & source_url, ContextPtr context_)
+    : IDatabase(name_)
+    , WithContext(context_->getGlobalContext())
+    , source(source_url)
+    , log(&Poco::Logger::get("DatabaseHDFS(" + name_ + ")"))
+{
+    if (!source.empty())
+    {
+        if (!re2::RE2::FullMatch(source, std::string(HDFS_HOST_REGEXP)))
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs host: {}. "
+                            "It should have structure 'hdfs://<host_name>:<port>'", source);
+
+        context_->getGlobalContext()->getRemoteHostFilter().checkURL(Poco::URI(source));
+    }
+}
+
+void DatabaseHDFS::addTable(const std::string & table_name, StoragePtr table_storage) const
+{
+    std::lock_guard lock(mutex);
+    auto [_, inserted] = loaded_tables.emplace(table_name, table_storage);
+    if (!inserted)
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Table with name `{}` already exists in database `{}` (engine {})",
+            table_name, getDatabaseName(), getEngineName());
+}
+
+std::string DatabaseHDFS::getTablePath(const std::string & table_name) const
+{
+    if (table_name.starts_with("hdfs://"))
+        return table_name;
+
+    if (source.empty())
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}. "
+                        "It should have structure 'hdfs://<host_name>:<port>/path'", table_name);
+
+    return fs::path(source) / table_name;
+}
+
+bool DatabaseHDFS::checkUrl(const std::string & url, ContextPtr context_, bool throw_on_error) const
+{
+    try
+    {
+        checkHDFSURL(url);
+        context_->getGlobalContext()->getRemoteHostFilter().checkURL(Poco::URI(url));
+    }
+    catch (...)
+    {
+        if (throw_on_error)
+            throw;
+        return false;
+    }
+
+    return true;
+}
+
+bool DatabaseHDFS::isTableExist(const String & name, ContextPtr context_) const
+{
+    std::lock_guard lock(mutex);
+    if (loaded_tables.find(name) != loaded_tables.end())
+        return true;
+
+    return checkUrl(name, context_, false);
+}
+
+StoragePtr DatabaseHDFS::getTableImpl(const String & name, ContextPtr context_) const
+{
+    /// Check if the table exists in the loaded tables map.
+    {
+        std::lock_guard lock(mutex);
+        auto it = loaded_tables.find(name);
+        if (it != loaded_tables.end())
+            return it->second;
+    }
+
+    auto url = getTablePath(name);
+
+    checkUrl(url, context_, true);
+
+    auto args = makeASTFunction("hdfs", std::make_shared<ASTLiteral>(url));
+
+    auto table_function = TableFunctionFactory::instance().get(args, context_);
+    if (!table_function)
+        return nullptr;
+
+    /// TableFunctionHDFS throws exceptions, if table cannot be created.
+    auto table_storage = table_function->execute(args, context_, name);
+    if (table_storage)
+        addTable(name, table_storage);
+
+    return table_storage;
+}
+
+StoragePtr DatabaseHDFS::getTable(const String & name, ContextPtr context_) const
+{
+    /// Rethrow all exceptions from TableFunctionHDFS to show correct error to user.
+    if (auto storage = getTableImpl(name, context_))
+        return storage;
+
+    throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist",
+                    backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name));
+}
+
+StoragePtr DatabaseHDFS::tryGetTable(const String & name, ContextPtr context_) const
+{
+    try
+    {
+        return getTableImpl(name, context_);
+    }
+    catch (const Exception & e)
+    {
+        // Ignore exceptions thrown by TableFunctionHDFS, which indicate that there is no table
+        if (e.code() == ErrorCodes::BAD_ARGUMENTS
+            || e.code() == ErrorCodes::ACCESS_DENIED
+            || e.code() == ErrorCodes::DATABASE_ACCESS_DENIED
+            || e.code() == ErrorCodes::FILE_DOESNT_EXIST
+            || e.code() == ErrorCodes::UNACCEPTABLE_URL
+            || e.code() == ErrorCodes::HDFS_ERROR
+            || e.code() == ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE)
+        {
+            return nullptr;
+        }
+        throw;
+    }
+    catch (const Poco::URISyntaxException &)
+    {
+        return nullptr;
+    }
+}
+
+bool DatabaseHDFS::empty() const
+{
+    std::lock_guard lock(mutex);
+    return loaded_tables.empty();
+}
+
+ASTPtr DatabaseHDFS::getCreateDatabaseQuery() const
+{
+    const auto & settings = getContext()->getSettingsRef();
+    ParserCreateQuery parser;
+
+    const String query = fmt::format("CREATE DATABASE {} ENGINE = HDFS('{}')", backQuoteIfNeed(getDatabaseName()), source);
+    ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
+
+    if (const auto database_comment = getDatabaseComment(); !database_comment.empty())
+    {
+        auto & ast_create_query = ast->as<ASTCreateQuery &>();
+        ast_create_query.set(ast_create_query.comment, std::make_shared<ASTLiteral>(database_comment));
+    }
+
+    return ast;
+}
+
+void DatabaseHDFS::shutdown()
+{
+    Tables tables_snapshot;
+    {
+        std::lock_guard lock(mutex);
+        tables_snapshot = loaded_tables;
+    }
+
+    for (const auto & kv : tables_snapshot)
+    {
+        auto table_id = kv.second->getStorageID();
+        kv.second->flushAndShutdown();
+    }
+
+    std::lock_guard lock(mutex);
+    loaded_tables.clear();
+}
+
+/**
+ * Returns an empty vector because the database is read-only and no tables can be backed up
+ */
+std::vector<std::pair<ASTPtr, StoragePtr>> DatabaseHDFS::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const
+{
+    return {};
+}
+
+/**
+ *
+ * Returns an empty iterator because the database does not have its own tables
+ * But only caches them for quick access
+ */
+DatabaseTablesIteratorPtr DatabaseHDFS::getTablesIterator(ContextPtr, const FilterByNameFunction &) const
+{
+    return std::make_unique<DatabaseTablesSnapshotIterator>(Tables{}, getDatabaseName());
+}
+
+} // DB
+
+#endif

src/Databases/DatabaseHDFS.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include "config.h"
+
+#if USE_HDFS
+
+#include <mutex>
+#include <Databases/IDatabase.h>
+#include <Parsers/IAST.h>
+#include <Storages/IStorage_fwd.h>
+#include <base/types.h>
+
+namespace DB
+{
+
+class Context;
+
+/**
+  * DatabaseHDFS allows to interact with files stored on the file system.
+  * Uses TableFunctionHDFS to implicitly load file when a user requests the table,
+  * and provides read-only access to the data in the file.
+  * Tables are cached inside the database for quick access.
+  */
+class DatabaseHDFS : public IDatabase, protected WithContext
+{
+public:
+    DatabaseHDFS(const String & name, const String & source_url, ContextPtr context);
+
+    String getEngineName() const override { return "S3"; }
+
+    bool isTableExist(const String & name, ContextPtr context) const override;
+
+    StoragePtr getTable(const String & name, ContextPtr context) const override;
+
+    StoragePtr tryGetTable(const String & name, ContextPtr context) const override;
+
+    bool shouldBeEmptyOnDetach() const override { return false; } /// Contains only temporary tables.
+
+    bool empty() const override;
+
+    bool isReadOnly() const override { return true; }
+
+    ASTPtr getCreateDatabaseQuery() const override;
+
+    void shutdown() override;
+
+    std::vector<std::pair<ASTPtr, StoragePtr>> getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const override;
+    DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &) const override;
+
+protected:
+    StoragePtr getTableImpl(const String & name, ContextPtr context) const;
+
+    void addTable(const std::string & table_name, StoragePtr table_storage) const;
+
+    bool checkUrl(const std::string & url, ContextPtr context_, bool throw_on_error) const;
+
+    std::string getTablePath(const std::string & table_name) const;
+
+private:
+    const String source;
+
+    mutable Tables loaded_tables TSA_GUARDED_BY(mutex);
+    Poco::Logger * log;
+};
+
+}
+
+#endif

src/Databases/DatabaseS3.cpp

@@ -0,0 +1,312 @@
+#include "config.h"
+
+#if USE_AWS_S3
+
+#include <Databases/DatabaseS3.h>
+
+#include <Interpreters/Context.h>
+#include <Interpreters/evaluateConstantExpression.h>
+#include <IO/S3/URI.h>
+#include <Parsers/ASTCreateQuery.h>
+#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/parseQuery.h>
+#include <Parsers/ParserCreateQuery.h>
+#include <Storages/checkAndGetLiteralArgument.h>
+#include <Storages/IStorage.h>
+#include <Storages/NamedCollectionsHelpers.h>
+#include <TableFunctions/TableFunctionFactory.h>
+
+#include <boost/algorithm/string.hpp>
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+namespace DB
+{
+
+static const std::unordered_set<std::string_view> optional_configuration_keys = {
+    "url",
+    "access_key_id",
+    "secret_access_key",
+    "no_sign_request"
+};
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int UNKNOWN_TABLE;
+    extern const int BAD_ARGUMENTS;
+    extern const int FILE_DOESNT_EXIST;
+    extern const int UNACCEPTABLE_URL;
+    extern const int S3_ERROR;
+
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+}
+
+DatabaseS3::DatabaseS3(const String & name_, const Configuration& config_, ContextPtr context_)
+    : IDatabase(name_)
+    , WithContext(context_->getGlobalContext())
+    , config(config_)
+    , log(&Poco::Logger::get("DatabaseS3(" + name_ + ")"))
+{
+}
+
+void DatabaseS3::addTable(const std::string & table_name, StoragePtr table_storage) const
+{
+    std::lock_guard lock(mutex);
+    auto [_, inserted] = loaded_tables.emplace(table_name, table_storage);
+    if (!inserted)
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "Table with name `{}` already exists in database `{}` (engine {})",
+            table_name, getDatabaseName(), getEngineName());
+}
+
+std::string DatabaseS3::getFullUrl(const std::string & name) const
+{
+    if (!config.url_prefix.empty())
+        return fs::path(config.url_prefix) / name;
+
+    return name;
+}
+
+bool DatabaseS3::checkUrl(const std::string & url, ContextPtr context_, bool throw_on_error) const
+{
+    try
+    {
+        S3::URI uri(url);
+        context_->getGlobalContext()->getRemoteHostFilter().checkURL(uri.uri);
+    }
+    catch (...)
+    {
+        if (throw_on_error)
+            throw;
+        return false;
+    }
+    return true;
+}
+
+bool DatabaseS3::isTableExist(const String & name, ContextPtr context_) const
+{
+    std::lock_guard lock(mutex);
+    if (loaded_tables.find(name) != loaded_tables.end())
+        return true;
+
+    return checkUrl(getFullUrl(name), context_, false);
+}
+
+StoragePtr DatabaseS3::getTableImpl(const String & name, ContextPtr context_) const
+{
+    /// Check if the table exists in the loaded tables map.
+    {
+        std::lock_guard lock(mutex);
+        auto it = loaded_tables.find(name);
+        if (it != loaded_tables.end())
+            return it->second;
+    }
+
+    auto url = getFullUrl(name);
+    checkUrl(url, context_, /* throw_on_error */true);
+
+    auto function = std::make_shared<ASTFunction>();
+    function->name = "s3";
+    function->arguments = std::make_shared<ASTExpressionList>();
+    function->children.push_back(function->arguments);
+
+    function->arguments->children.push_back(std::make_shared<ASTLiteral>(url));
+    if (config.no_sign_request)
+    {
+        function->arguments->children.push_back(std::make_shared<ASTLiteral>("NOSIGN"));
+    }
+    else if (config.access_key_id.has_value() && config.secret_access_key.has_value())
+    {
+        function->arguments->children.push_back(std::make_shared<ASTLiteral>(config.access_key_id.value()));
+        function->arguments->children.push_back(std::make_shared<ASTLiteral>(config.secret_access_key.value()));
+    }
+
+    auto table_function = TableFunctionFactory::instance().get(function, context_);
+    if (!table_function)
+        return nullptr;
+
+    /// TableFunctionS3 throws exceptions, if table cannot be created.
+    auto table_storage = table_function->execute(function, context_, name);
+    if (table_storage)
+        addTable(name, table_storage);
+
+    return table_storage;
+}
+
+StoragePtr DatabaseS3::getTable(const String & name, ContextPtr context_) const
+{
+    /// Rethrow all exceptions from TableFunctionS3 to show correct error to user.
+    if (auto storage = getTableImpl(name, context_))
+        return storage;
+
+    throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist",
+                    backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name));
+}
+
+StoragePtr DatabaseS3::tryGetTable(const String & name, ContextPtr context_) const
+{
+    try
+    {
+        return getTableImpl(name, context_);
+    }
+    catch (const Exception & e)
+    {
+        /// Ignore exceptions thrown by TableFunctionS3, which indicate that there is no table.
+        if (e.code() == ErrorCodes::BAD_ARGUMENTS
+            || e.code() == ErrorCodes::S3_ERROR
+            || e.code() == ErrorCodes::FILE_DOESNT_EXIST
+            || e.code() == ErrorCodes::UNACCEPTABLE_URL)
+        {
+            return nullptr;
+        }
+        throw;
+    }
+    catch (const Poco::URISyntaxException &)
+    {
+        return nullptr;
+    }
+}
+
+bool DatabaseS3::empty() const
+{
+    std::lock_guard lock(mutex);
+    return loaded_tables.empty();
+}
+
+ASTPtr DatabaseS3::getCreateDatabaseQuery() const
+{
+    const auto & settings = getContext()->getSettingsRef();
+    ParserCreateQuery parser;
+
+    std::string creation_args;
+    creation_args += fmt::format("'{}'", config.url_prefix);
+    if (config.no_sign_request)
+        creation_args += ", 'NOSIGN'";
+    else if (config.access_key_id.has_value() && config.secret_access_key.has_value())
+        creation_args += fmt::format(", '{}', '{}'", config.access_key_id.value(), config.secret_access_key.value());
+
+    const String query = fmt::format("CREATE DATABASE {} ENGINE = S3({})", backQuoteIfNeed(getDatabaseName()), creation_args);
+    ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
+
+    if (const auto database_comment = getDatabaseComment(); !database_comment.empty())
+    {
+        auto & ast_create_query = ast->as<ASTCreateQuery &>();
+        ast_create_query.set(ast_create_query.comment, std::make_shared<ASTLiteral>(database_comment));
+    }
+
+    return ast;
+}
+
+void DatabaseS3::shutdown()
+{
+    Tables tables_snapshot;
+    {
+        std::lock_guard lock(mutex);
+        tables_snapshot = loaded_tables;
+    }
+
+    for (const auto & kv : tables_snapshot)
+    {
+        auto table_id = kv.second->getStorageID();
+        kv.second->flushAndShutdown();
+    }
+
+    std::lock_guard lock(mutex);
+    loaded_tables.clear();
+}
+
+DatabaseS3::Configuration DatabaseS3::parseArguments(ASTs engine_args, ContextPtr context_)
+{
+    Configuration result;
+
+    if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args, context_))
+    {
+        auto & collection = *named_collection;
+
+        validateNamedCollection(collection, {}, optional_configuration_keys);
+
+        result.url_prefix = collection.getOrDefault<String>("url", "");
+        result.no_sign_request = collection.getOrDefault<bool>("no_sign_request", false);
+
+        auto key_id = collection.getOrDefault<String>("access_key_id", "");
+        auto secret_key = collection.getOrDefault<String>("secret_access_key", "");
+
+        if (!key_id.empty())
+            result.access_key_id = key_id;
+
+        if (!secret_key.empty())
+            result.secret_access_key = secret_key;
+    }
+    else
+    {
+        const std::string supported_signature =
+            " - S3()\n"
+            " - S3('url')\n"
+            " - S3('url', 'NOSIGN')\n"
+            " - S3('url', 'access_key_id', 'secret_access_key')\n";
+        const auto error_message =
+            fmt::format("Engine DatabaseS3 must have the following arguments signature\n{}", supported_signature);
+
+        for (auto & arg : engine_args)
+            arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context_);
+
+        if (engine_args.size() > 3)
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, error_message.c_str());
+
+        if (engine_args.empty())
+            return result;
+
+        result.url_prefix = checkAndGetLiteralArgument<String>(engine_args[0], "url");
+
+        // url, NOSIGN
+        if (engine_args.size() == 2)
+        {
+            auto second_arg = checkAndGetLiteralArgument<String>(engine_args[1], "NOSIGN");
+            if (boost::iequals(second_arg, "NOSIGN"))
+                result.no_sign_request = true;
+            else
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, error_message.c_str());
+        }
+
+        // url, access_key_id, secret_access_key
+        if (engine_args.size() == 3)
+        {
+            auto key_id = checkAndGetLiteralArgument<String>(engine_args[1], "access_key_id");
+            auto secret_key = checkAndGetLiteralArgument<String>(engine_args[2], "secret_access_key");
+
+            if (key_id.empty() || secret_key.empty() || boost::iequals(key_id, "NOSIGN"))
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, error_message.c_str());
+
+            result.access_key_id = key_id;
+            result.secret_access_key = secret_key;
+        }
+    }
+
+    return result;
+}
+
+/**
+ * Returns an empty vector because the database is read-only and no tables can be backed up
+ */
+std::vector<std::pair<ASTPtr, StoragePtr>> DatabaseS3::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const
+{
+    return {};
+}
+
+/**
+ *
+ * Returns an empty iterator because the database does not have its own tables
+ * But only caches them for quick access
+ */
+DatabaseTablesIteratorPtr DatabaseS3::getTablesIterator(ContextPtr, const FilterByNameFunction &) const
+{
+    return std::make_unique<DatabaseTablesSnapshotIterator>(Tables{}, getDatabaseName());
+}
+
+}
+
+#endif

src/Databases/DatabaseS3.h

@@ -0,0 +1,81 @@
+#pragma once
+
+#include "config.h"
+
+#if USE_AWS_S3
+
+#include <mutex>
+#include <Databases/IDatabase.h>
+#include <Parsers/IAST.h>
+#include <Storages/IStorage_fwd.h>
+#include <base/types.h>
+
+namespace DB
+{
+
+class Context;
+
+/**
+  * DatabaseS3 provides access to data stored in S3.
+  * Uses TableFunctionS3 to implicitly load file when a user requests the table,
+  * and provides read-only access to the data in the file.
+  * Tables are cached inside the database for quick access.
+  */
+class DatabaseS3 : public IDatabase, protected WithContext
+{
+public:
+    struct Configuration
+    {
+        std::string url_prefix;
+
+        bool no_sign_request = false;
+
+        std::optional<std::string> access_key_id;
+        std::optional<std::string> secret_access_key;
+    };
+
+    DatabaseS3(const String & name, const Configuration& config, ContextPtr context);
+
+    String getEngineName() const override { return "S3"; }
+
+    bool isTableExist(const String & name, ContextPtr context) const override;
+
+    StoragePtr getTable(const String & name, ContextPtr context) const override;
+
+    StoragePtr tryGetTable(const String & name, ContextPtr context) const override;
+
+    // Contains only temporary tables
+    bool shouldBeEmptyOnDetach() const override { return false; }
+
+    bool empty() const override;
+
+    bool isReadOnly() const override { return true; }
+
+    ASTPtr getCreateDatabaseQuery() const override;
+
+    void shutdown() override;
+
+    std::vector<std::pair<ASTPtr, StoragePtr>> getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const override;
+    DatabaseTablesIteratorPtr getTablesIterator(ContextPtr, const FilterByNameFunction &) const override;
+
+    static Configuration parseArguments(ASTs engine_args, ContextPtr context);
+
+protected:
+    StoragePtr getTableImpl(const String & name, ContextPtr context) const;
+
+    void addTable(const std::string & table_name, StoragePtr table_storage) const;
+
+    bool checkUrl(const std::string & url, ContextPtr context_, bool throw_on_error) const;
+
+    std::string getFullUrl(const std::string & name) const;
+
+private:
+    const Configuration config;
+
+    mutable Tables loaded_tables TSA_GUARDED_BY(mutex);
+    Poco::Logger * log;
+};
+
+}
+
+#endif

src/Databases/DatabasesOverlay.cpp

@@ -0,0 +1,266 @@
+#include <Databases/DatabasesOverlay.h>
+
+#include <Common/typeid_cast.h>
+#include <Interpreters/Context.h>
+#include <Interpreters/InterpreterCreateQuery.h>
+#include <Parsers/ASTCreateQuery.h>
+
+#include <Storages/IStorage_fwd.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int CANNOT_GET_CREATE_TABLE_QUERY;
+}
+
+DatabasesOverlay::DatabasesOverlay(const String & name_, ContextPtr context_)
+    : IDatabase(name_), WithContext(context_->getGlobalContext()), log(&Poco::Logger::get("DatabaseOverlay(" + name_ + ")"))
+{
+}
+
+DatabasesOverlay & DatabasesOverlay::registerNextDatabase(DatabasePtr database)
+{
+    databases.push_back(std::move(database));
+    return *this;
+}
+
+bool DatabasesOverlay::isTableExist(const String & table_name, ContextPtr context_) const
+{
+    for (const auto & db : databases)
+    {
+        if (db->isTableExist(table_name, context_))
+            return true;
+    }
+    return false;
+}
+
+StoragePtr DatabasesOverlay::tryGetTable(const String & table_name, ContextPtr context_) const
+{
+    StoragePtr result = nullptr;
+    for (const auto & db : databases)
+    {
+        result = db->tryGetTable(table_name, context_);
+        if (result)
+            break;
+    }
+    return result;
+}
+
+void DatabasesOverlay::createTable(ContextPtr context_, const String & table_name, const StoragePtr & table, const ASTPtr & query)
+{
+    for (auto & db : databases)
+    {
+        if (!db->isReadOnly())
+        {
+            db->createTable(context_, table_name, table, query);
+            return;
+        }
+    }
+    throw Exception(
+        ErrorCodes::LOGICAL_ERROR,
+        "There is no databases for CREATE TABLE `{}` query in database `{}` (engine {})",
+        table_name,
+        getDatabaseName(),
+        getEngineName());
+}
+
+void DatabasesOverlay::dropTable(ContextPtr context_, const String & table_name, bool sync)
+{
+    for (auto & db : databases)
+    {
+        if (db->isTableExist(table_name, context_))
+        {
+            db->dropTable(context_, table_name, sync);
+            return;
+        }
+    }
+    throw Exception(
+        ErrorCodes::LOGICAL_ERROR,
+        "There is no databases for DROP TABLE `{}` query in database `{}` (engine {})",
+        table_name,
+        getDatabaseName(),
+        getEngineName());
+}
+
+void DatabasesOverlay::attachTable(
+    ContextPtr context_, const String & table_name, const StoragePtr & table, const String & relative_table_path)
+{
+    for (auto & db : databases)
+    {
+        try
+        {
+            db->attachTable(context_, table_name, table, relative_table_path);
+            return;
+        }
+        catch (...)
+        {
+            continue;
+        }
+    }
+    throw Exception(
+        ErrorCodes::LOGICAL_ERROR,
+        "There is no databases for ATTACH TABLE `{}` query in database `{}` (engine {})",
+        table_name,
+        getDatabaseName(),
+        getEngineName());
+}
+
+StoragePtr DatabasesOverlay::detachTable(ContextPtr context_, const String & table_name)
+{
+    StoragePtr result = nullptr;
+    for (auto & db : databases)
+    {
+        if (db->isTableExist(table_name, context_))
+            return db->detachTable(context_, table_name);
+    }
+    throw Exception(
+        ErrorCodes::LOGICAL_ERROR,
+        "There is no databases for DETACH TABLE `{}` query in database `{}` (engine {})",
+        table_name,
+        getDatabaseName(),
+        getEngineName());
+}
+
+ASTPtr DatabasesOverlay::getCreateTableQueryImpl(const String & name, ContextPtr context_, bool throw_on_error) const
+{
+    ASTPtr result = nullptr;
+    for (const auto & db : databases)
+    {
+        result = db->tryGetCreateTableQuery(name, context_);
+        if (result)
+            break;
+    }
+    if (!result && throw_on_error)
+        throw Exception(
+            ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY,
+            "There is no metadata of table `{}` in database `{}` (engine {})",
+            name,
+            getDatabaseName(),
+            getEngineName());
+    return result;
+}
+
+/*
+ * DatabaseOverlay cannot be constructed by "CREATE DATABASE" query, as it is not a traditional ClickHouse database
+ * To use DatabaseOverlay, it must be constructed programmatically in code
+ */
+ASTPtr DatabasesOverlay::getCreateDatabaseQuery() const
+{
+    return std::make_shared<ASTCreateQuery>();
+}
+
+String DatabasesOverlay::getTableDataPath(const String & table_name) const
+{
+    String result;
+    for (const auto & db : databases)
+    {
+        result = db->getTableDataPath(table_name);
+        if (!result.empty())
+            break;
+    }
+    return result;
+}
+
+String DatabasesOverlay::getTableDataPath(const ASTCreateQuery & query) const
+{
+    String result;
+    for (const auto & db : databases)
+    {
+        result = db->getTableDataPath(query);
+        if (!result.empty())
+            break;
+    }
+    return result;
+}
+
+UUID DatabasesOverlay::tryGetTableUUID(const String & table_name) const
+{
+    UUID result = UUIDHelpers::Nil;
+    for (const auto & db : databases)
+    {
+        result = db->tryGetTableUUID(table_name);
+        if (result != UUIDHelpers::Nil)
+            break;
+    }
+    return result;
+}
+
+void DatabasesOverlay::drop(ContextPtr context_)
+{
+    for (auto & db : databases)
+        db->drop(context_);
+}
+
+void DatabasesOverlay::alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata)
+{
+    for (auto & db : databases)
+    {
+        if (!db->isReadOnly() && db->isTableExist(table_id.table_name, local_context))
+        {
+            db->alterTable(local_context, table_id, metadata);
+            return;
+        }
+    }
+    throw Exception(
+        ErrorCodes::LOGICAL_ERROR,
+        "There is no databases for ALTER TABLE `{}` query in database `{}` (engine {})",
+        table_id.table_name,
+        getDatabaseName(),
+        getEngineName());
+}
+
+std::vector<std::pair<ASTPtr, StoragePtr>>
+DatabasesOverlay::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const
+{
+    std::vector<std::pair<ASTPtr, StoragePtr>> result;
+    for (const auto & db : databases)
+    {
+        auto db_backup = db->getTablesForBackup(filter, local_context);
+        result.insert(result.end(), std::make_move_iterator(db_backup.begin()), std::make_move_iterator(db_backup.end()));
+    }
+    return result;
+}
+
+void DatabasesOverlay::createTableRestoredFromBackup(
+    const ASTPtr & create_table_query,
+    ContextMutablePtr local_context,
+    std::shared_ptr<IRestoreCoordination> /*restore_coordination*/,
+    UInt64 /*timeout_ms*/)
+{
+    /// Creates a tables by executing a "CREATE TABLE" query.
+    InterpreterCreateQuery interpreter{create_table_query, local_context};
+    interpreter.setInternal(true);
+    interpreter.execute();
+}
+
+bool DatabasesOverlay::empty() const
+{
+    for (const auto & db : databases)
+    {
+        if (!db->empty())
+            return false;
+    }
+    return true;
+}
+
+void DatabasesOverlay::shutdown()
+{
+    for (auto & db : databases)
+        db->shutdown();
+}
+
+DatabaseTablesIteratorPtr DatabasesOverlay::getTablesIterator(ContextPtr context_, const FilterByNameFunction & filter_by_table_name) const
+{
+    Tables tables;
+    for (const auto & db : databases)
+    {
+        for (auto table_it = db->getTablesIterator(context_, filter_by_table_name); table_it->isValid(); table_it->next())
+            tables.insert({table_it->name(), table_it->table()});
+    }
+    return std::make_unique<DatabaseTablesSnapshotIterator>(std::move(tables), getDatabaseName());
+}
+
+}

src/Databases/DatabasesOverlay.h

@@ -0,0 +1,66 @@
+#pragma once
+
+#include <Storages/IStorage_fwd.h>
+#include <Databases/IDatabase.h>
+
+namespace DB
+{
+
+/**
+ * Implements the IDatabase interface and combines multiple other databases
+ * Searches for tables in each database in order until found, and delegates operations to the appropriate database
+ * Useful for combining databases
+ *
+ * Used in clickhouse-local to combine DatabaseFileSystem and DatabaseMemory
+ */
+class DatabasesOverlay : public IDatabase, protected WithContext
+{
+public:
+    DatabasesOverlay(const String & name_, ContextPtr context_);
+
+    /// Not thread-safe. Use only as factory to initialize database
+    DatabasesOverlay & registerNextDatabase(DatabasePtr database);
+
+    String getEngineName() const override { return "Overlay"; }
+
+public:
+    bool isTableExist(const String & table_name, ContextPtr context) const override;
+
+    StoragePtr tryGetTable(const String & table_name, ContextPtr context) const override;
+
+    void createTable(ContextPtr context, const String & table_name, const StoragePtr & table, const ASTPtr & query) override;
+
+    void dropTable(ContextPtr context, const String & table_name, bool sync) override;
+
+    void attachTable(ContextPtr context, const String & table_name, const StoragePtr & table, const String & relative_table_path) override;
+
+    StoragePtr detachTable(ContextPtr context, const String & table_name) override;
+
+    ASTPtr getCreateTableQueryImpl(const String & name, ContextPtr context, bool throw_on_error) const override;
+    ASTPtr getCreateDatabaseQuery() const override;
+
+    String getTableDataPath(const String & table_name) const override;
+    String getTableDataPath(const ASTCreateQuery & query) const override;
+
+    UUID tryGetTableUUID(const String & table_name) const override;
+
+    void drop(ContextPtr context) override;
+
+    void alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) override;
+
+    std::vector<std::pair<ASTPtr, StoragePtr>> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override;
+
+    void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr<IRestoreCoordination> restore_coordination, UInt64 timeout_ms) override;
+
+    DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;
+
+    bool empty() const override;
+
+    void shutdown() override;
+
+protected:
+    std::vector<DatabasePtr> databases;
+    Poco::Logger * log;
+};
+
+}

src/Databases/IDatabase.h

@@ -170,7 +170,7 @@ public:
     /// Get the table for work. Return nullptr if there is no table.
     virtual StoragePtr tryGetTable(const String & name, ContextPtr context) const = 0;
 
-    StoragePtr getTable(const String & name, ContextPtr context) const;
+    virtual StoragePtr getTable(const String & name, ContextPtr context) const;
 
     virtual UUID tryGetTableUUID(const String & /*table_name*/) const { return UUIDHelpers::Nil; }
 
@@ -183,6 +183,8 @@ public:
     /// Is the database empty.
     virtual bool empty() const = 0;
 
+    virtual bool isReadOnly() const { return false; }
+
     /// Add the table to the database. Record its presence in the metadata.
     virtual void createTable(
         ContextPtr /*context*/,

src/Dictionaries/ClickHouseDictionarySource.cpp

@@ -217,7 +217,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory)
         std::optional<Configuration> configuration;
 
         std::string settings_config_prefix = config_prefix + ".clickhouse";
-        auto named_collection = created_from_ddl ? tryGetNamedCollectionWithOverrides(config, settings_config_prefix) : nullptr;
+        auto named_collection = created_from_ddl ? tryGetNamedCollectionWithOverrides(config, settings_config_prefix, global_context) : nullptr;
 
         if (named_collection)
         {

src/Dictionaries/MySQLDictionarySource.cpp

@@ -71,7 +71,7 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory)
         MySQLSettings mysql_settings;
 
         std::optional<MySQLDictionarySource::Configuration> dictionary_configuration;
-        auto named_collection = created_from_ddl ? tryGetNamedCollectionWithOverrides(config, settings_config_prefix) : nullptr;
+        auto named_collection = created_from_ddl ? tryGetNamedCollectionWithOverrides(config, settings_config_prefix, global_context) : nullptr;
         if (named_collection)
         {
             auto allowed_arguments{dictionary_allowed_keys};

src/Dictionaries/RegExpTreeDictionary.cpp

@@ -30,8 +30,6 @@
 #include <Dictionaries/RegExpTreeDictionary.h>
 #include <Dictionaries/YAMLRegExpTreeDictionarySource.h>
 
-#include <re2_st/stringpiece.h>
-
 #include "config.h"
 
 #if USE_VECTORSCAN
@@ -469,10 +467,9 @@ public:
 
 std::pair<String, bool> processBackRefs(const String & data, const re2_st::RE2 & searcher, const std::vector<StringPiece> & pieces)
 {
-    re2_st::StringPiece haystack(data.data(), data.size());
-    re2_st::StringPiece matches[10];
+    std::string_view matches[10];
     String result;
-    searcher.Match(haystack, 0, data.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 10);
+    searcher.Match({data.data(), data.size()}, 0, data.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 10);
     /// if the pattern is a single '$1' but fails to match, we would use the default value.
     if (pieces.size() == 1 && pieces[0].ref_num >= 0 && pieces[0].ref_num < 10 && matches[pieces[0].ref_num].empty())
         return std::make_pair(result, true);

src/Disks/DiskEncrypted.cpp

@@ -285,19 +285,32 @@ private:
 };
 
 DiskEncrypted::DiskEncrypted(
-    const String & name_, const Poco::Util::AbstractConfiguration & config_, const String & config_prefix_, const DisksMap & map_, bool use_fake_transaction_)
-    : DiskEncrypted(name_, parseDiskEncryptedSettings(name_, config_, config_prefix_, map_), use_fake_transaction_)
+    const String & name_, const Poco::Util::AbstractConfiguration & config_, const String & config_prefix_, const DisksMap & map_)
+    : DiskEncrypted(name_, parseDiskEncryptedSettings(name_, config_, config_prefix_, map_), config_, config_prefix_)
 {
 }
 
-DiskEncrypted::DiskEncrypted(const String & name_, std::unique_ptr<const DiskEncryptedSettings> settings_, bool use_fake_transaction_)
+DiskEncrypted::DiskEncrypted(const String & name_, std::unique_ptr<const DiskEncryptedSettings> settings_,
+                             const Poco::Util::AbstractConfiguration & config_, const String & config_prefix_)
+    : IDisk(name_, config_, config_prefix_)
+    , delegate(settings_->wrapped_disk)
+    , encrypted_name(name_)
+    , disk_path(settings_->disk_path)
+    , disk_absolute_path(settings_->wrapped_disk->getPath() + settings_->disk_path)
+    , current_settings(std::move(settings_))
+    , use_fake_transaction(config_.getBool(config_prefix_ + ".use_fake_transaction", true))
+{
+    delegate->createDirectories(disk_path);
+}
+
+DiskEncrypted::DiskEncrypted(const String & name_, std::unique_ptr<const DiskEncryptedSettings> settings_)
     : IDisk(name_)
     , delegate(settings_->wrapped_disk)
     , encrypted_name(name_)
     , disk_path(settings_->disk_path)
     , disk_absolute_path(settings_->wrapped_disk->getPath() + settings_->disk_path)
     , current_settings(std::move(settings_))
-    , use_fake_transaction(use_fake_transaction_)
+    , use_fake_transaction(true)
 {
     delegate->createDirectories(disk_path);
 }
@@ -310,32 +323,6 @@ ReservationPtr DiskEncrypted::reserve(UInt64 bytes)
     return std::make_unique<DiskEncryptedReservation>(std::static_pointer_cast<DiskEncrypted>(shared_from_this()), std::move(reservation));
 }
 
-void DiskEncrypted::copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path)
-{
-    /// Check if we can copy the file without deciphering.
-    if (isSameDiskType(*this, *to_disk))
-    {
-        /// Disk type is the same, check if the key is the same too.
-        if (auto * to_disk_enc = typeid_cast<DiskEncrypted *>(to_disk.get()))
-        {
-            auto from_settings = current_settings.get();
-            auto to_settings = to_disk_enc->current_settings.get();
-            if (from_settings->all_keys == to_settings->all_keys)
-            {
-                /// Keys are the same so we can simply copy the encrypted file.
-                auto wrapped_from_path = wrappedPath(from_path);
-                auto to_delegate = to_disk_enc->delegate;
-                auto wrapped_to_path = to_disk_enc->wrappedPath(to_path);
-                delegate->copy(wrapped_from_path, to_delegate, wrapped_to_path);
-                return;
-            }
-        }
-    }
-
-    /// Copy the file through buffers with deciphering.
-    copyThroughBuffers(from_path, to_disk, to_path);
-}
-
 
 void DiskEncrypted::copyDirectoryContent(const String & from_dir, const std::shared_ptr<IDisk> & to_disk, const String & to_dir)
 {
@@ -359,11 +346,8 @@ void DiskEncrypted::copyDirectoryContent(const String & from_dir, const std::sha
         }
     }
 
-    if (!to_disk->exists(to_dir))
-        to_disk->createDirectories(to_dir);
-
     /// Copy the file through buffers with deciphering.
-    copyThroughBuffers(from_dir, to_disk, to_dir);
+    IDisk::copyDirectoryContent(from_dir, to_disk, to_dir);
 }
 
 std::unique_ptr<ReadBufferFromFileBase> DiskEncrypted::readFile(
@@ -443,7 +427,7 @@ std::unordered_map<String, String> DiskEncrypted::getSerializedMetadata(const st
 
 void DiskEncrypted::applyNewSettings(
     const Poco::Util::AbstractConfiguration & config,
-    ContextPtr /*context*/,
+    ContextPtr context,
     const String & config_prefix,
     const DisksMap & disk_map)
 {
@@ -455,6 +439,7 @@ void DiskEncrypted::applyNewSettings(
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Сhanging disk path on the fly is not supported. Disk {}", name);
 
     current_settings.set(std::move(new_settings));
+    IDisk::applyNewSettings(config, context, config_prefix, disk_map);
 }
 
 void registerDiskEncrypted(DiskFactory & factory, bool global_skip_access_check)
@@ -467,7 +452,7 @@ void registerDiskEncrypted(DiskFactory & factory, bool global_skip_access_check)
         const DisksMap & map) -> DiskPtr
     {
         bool skip_access_check = global_skip_access_check || config.getBool(config_prefix + ".skip_access_check", false);
-        DiskPtr disk = std::make_shared<DiskEncrypted>(name, config, config_prefix, map, config.getBool(config_prefix + ".use_fake_transaction", true));
+        DiskPtr disk = std::make_shared<DiskEncrypted>(name, config, config_prefix, map);
         disk->startup(context, skip_access_check);
         return disk;
     };

src/Disks/DiskEncrypted.h

@@ -21,8 +21,10 @@ class WriteBufferFromFileBase;
 class DiskEncrypted : public IDisk
 {
 public:
-    DiskEncrypted(const String & name_, const Poco::Util::AbstractConfiguration & config_, const String & config_prefix_, const DisksMap & map_, bool use_fake_transaction_);
-    DiskEncrypted(const String & name_, std::unique_ptr<const DiskEncryptedSettings> settings_, bool use_fake_transaction_);
+    DiskEncrypted(const String & name_, const Poco::Util::AbstractConfiguration & config_, const String & config_prefix_, const DisksMap & map_);
+    DiskEncrypted(const String & name_, std::unique_ptr<const DiskEncryptedSettings> settings_,
+                  const Poco::Util::AbstractConfiguration & config_, const String & config_prefix_);
+    DiskEncrypted(const String & name_, std::unique_ptr<const DiskEncryptedSettings> settings_);
 
     const String & getName() const override { return encrypted_name; }
     const String & getPath() const override { return disk_absolute_path; }
@@ -110,8 +112,6 @@ public:
         delegate->listFiles(wrapped_path, file_names);
     }
 
-    void copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path) override;
-
     void copyDirectoryContent(const String & from_dir, const std::shared_ptr<IDisk> & to_disk, const String & to_dir) override;
 
     std::unique_ptr<ReadBufferFromFileBase> readFile(

src/Disks/DiskLocal.cpp

@@ -417,29 +417,12 @@ bool inline isSameDiskType(const IDisk & one, const IDisk & another)
     return typeid(one) == typeid(another);
 }
 
-void DiskLocal::copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path)
-{
-    if (isSameDiskType(*this, *to_disk))
-    {
-        fs::path to = fs::path(to_disk->getPath()) / to_path;
-        fs::path from = fs::path(disk_path) / from_path;
-        if (from_path.ends_with('/'))
-            from = from.parent_path();
-        if (fs::is_directory(from))
-            to /= from.filename();
-
-        fs::copy(from, to, fs::copy_options::recursive | fs::copy_options::overwrite_existing); /// Use more optimal way.
-    }
-    else
-        copyThroughBuffers(from_path, to_disk, to_path, /* copy_root_dir */ true); /// Base implementation.
-}
-
 void DiskLocal::copyDirectoryContent(const String & from_dir, const std::shared_ptr<IDisk> & to_disk, const String & to_dir)
 {
     if (isSameDiskType(*this, *to_disk))
-        fs::copy(from_dir, to_dir, fs::copy_options::recursive | fs::copy_options::overwrite_existing); /// Use more optimal way.
+        fs::copy(fs::path(disk_path) / from_dir, fs::path(to_disk->getPath()) / to_dir, fs::copy_options::recursive | fs::copy_options::overwrite_existing); /// Use more optimal way.
     else
-        copyThroughBuffers(from_dir, to_disk, to_dir, /* copy_root_dir */ false); /// Base implementation.
+        IDisk::copyDirectoryContent(from_dir, to_disk, to_dir);
 }
 
 SyncGuardPtr DiskLocal::getDirectorySyncGuard(const String & path) const
@@ -448,7 +431,7 @@ SyncGuardPtr DiskLocal::getDirectorySyncGuard(const String & path) const
 }
 
 
-void DiskLocal::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap &)
+void DiskLocal::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & disk_map)
 {
     String new_disk_path;
     UInt64 new_keep_free_space_bytes;
@@ -460,10 +443,13 @@ void DiskLocal::applyNewSettings(const Poco::Util::AbstractConfiguration & confi
 
     if (keep_free_space_bytes != new_keep_free_space_bytes)
         keep_free_space_bytes = new_keep_free_space_bytes;
+
+    IDisk::applyNewSettings(config, context, config_prefix, disk_map);
 }
 
-DiskLocal::DiskLocal(const String & name_, const String & path_, UInt64 keep_free_space_bytes_)
-    : IDisk(name_)
+DiskLocal::DiskLocal(const String & name_, const String & path_, UInt64 keep_free_space_bytes_,
+                     const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
+    : IDisk(name_, config, config_prefix)
     , disk_path(path_)
     , keep_free_space_bytes(keep_free_space_bytes_)
     , logger(&Poco::Logger::get("DiskLocal"))
@@ -472,13 +458,24 @@ DiskLocal::DiskLocal(const String & name_, const String & path_, UInt64 keep_fre
 }
 
 DiskLocal::DiskLocal(
-    const String & name_, const String & path_, UInt64 keep_free_space_bytes_, ContextPtr context, UInt64 local_disk_check_period_ms)
-    : DiskLocal(name_, path_, keep_free_space_bytes_)
+    const String & name_, const String & path_, UInt64 keep_free_space_bytes_, ContextPtr context,
+    const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
+    : DiskLocal(name_, path_, keep_free_space_bytes_, config, config_prefix)
 {
+    auto local_disk_check_period_ms = config.getUInt("local_disk_check_period_ms", 0);
     if (local_disk_check_period_ms > 0)
         disk_checker = std::make_unique<DiskLocalCheckThread>(this, context, local_disk_check_period_ms);
 }
 
+DiskLocal::DiskLocal(const String & name_, const String & path_)
+    : IDisk(name_)
+    , disk_path(path_)
+    , keep_free_space_bytes(0)
+    , logger(&Poco::Logger::get("DiskLocal"))
+    , data_source_description(getLocalDataSourceDescription(disk_path))
+{
+}
+
 DataSourceDescription DiskLocal::getDataSourceDescription() const
 {
     return data_source_description;
@@ -720,7 +717,7 @@ void registerDiskLocal(DiskFactory & factory, bool global_skip_access_check)
 
         bool skip_access_check = global_skip_access_check || config.getBool(config_prefix + ".skip_access_check", false);
         std::shared_ptr<IDisk> disk
-            = std::make_shared<DiskLocal>(name, path, keep_free_space_bytes, context, config.getUInt("local_disk_check_period_ms", 0));
+            = std::make_shared<DiskLocal>(name, path, keep_free_space_bytes, context, config, config_prefix);
         disk->startup(context, skip_access_check);
         return disk;
     };

src/Disks/DiskLocal.h

@@ -19,13 +19,17 @@ public:
     friend class DiskLocalCheckThread;
     friend class DiskLocalReservation;
 
-    DiskLocal(const String & name_, const String & path_, UInt64 keep_free_space_bytes_);
+    DiskLocal(const String & name_, const String & path_, UInt64 keep_free_space_bytes_,
+              const Poco::Util::AbstractConfiguration & config, const String & config_prefix);
     DiskLocal(
         const String & name_,
         const String & path_,
         UInt64 keep_free_space_bytes_,
         ContextPtr context,
-        UInt64 local_disk_check_period_ms);
+        const Poco::Util::AbstractConfiguration & config,
+        const String & config_prefix);
+
+    DiskLocal(const String & name_, const String & path_);
 
     const String & getPath() const override { return disk_path; }
 
@@ -63,8 +67,6 @@ public:
 
     void replaceFile(const String & from_path, const String & to_path) override;
 
-    void copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path) override;
-
     void copyDirectoryContent(const String & from_dir, const std::shared_ptr<IDisk> & to_disk, const String & to_dir) override;
 
     void listFiles(const String & path, std::vector<String> & file_names) const override;

src/Disks/DiskSelector.cpp

@@ -53,7 +53,7 @@ void DiskSelector::initialize(const Poco::Util::AbstractConfiguration & config,
         disks.emplace(
             default_disk_name,
             std::make_shared<DiskLocal>(
-                default_disk_name, context->getPath(), 0, context, config.getUInt("local_disk_check_period_ms", 0)));
+                default_disk_name, context->getPath(), 0, context, config, config_prefix));
     }
 
     is_initialized = true;

src/Disks/Executor.h

@@ -1,42 +0,0 @@
-#pragma once
-
-#include <future>
-#include <functional>
-
-namespace DB
-{
-
-/// Interface to run task asynchronously with possibility to wait for execution.
-class Executor
-{
-public:
-    virtual ~Executor() = default;
-    virtual std::future<void> execute(std::function<void()> task) = 0;
-};
-
-/// Executes task synchronously in case when disk doesn't support async operations.
-class SyncExecutor : public Executor
-{
-public:
-    SyncExecutor() = default;
-    std::future<void> execute(std::function<void()> task) override
-    {
-        auto promise = std::make_shared<std::promise<void>>();
-        try
-        {
-            task();
-            promise->set_value();
-        }
-        catch (...)
-        {
-            try
-            {
-                promise->set_exception(std::current_exception());
-            }
-            catch (...) { }
-        }
-        return promise->get_future();
-    }
-};
-
-}

src/Disks/IDisk.cpp

@@ -1,5 +1,4 @@
 #include "IDisk.h"
-#include "Disks/Executor.h"
 #include <IO/ReadBufferFromFileBase.h>
 #include <IO/WriteBufferFromFileBase.h>
 #include <IO/copyData.h>
@@ -80,18 +79,33 @@ UInt128 IDisk::getEncryptedFileIV(const String &) const
 
 using ResultsCollector = std::vector<std::future<void>>;
 
-void asyncCopy(IDisk & from_disk, String from_path, IDisk & to_disk, String to_path, Executor & exec, ResultsCollector & results, bool copy_root_dir, const WriteSettings & settings)
+void asyncCopy(IDisk & from_disk, String from_path, IDisk & to_disk, String to_path, ThreadPool & pool, ResultsCollector & results, bool copy_root_dir, const WriteSettings & settings)
 {
     if (from_disk.isFile(from_path))
     {
-        auto result = exec.execute(
-            [&from_disk, from_path, &to_disk, to_path, &settings]()
+        auto promise = std::make_shared<std::promise<void>>();
+        auto future = promise->get_future();
+
+        pool.scheduleOrThrowOnError(
+            [&from_disk, from_path, &to_disk, to_path, &settings, promise, thread_group = CurrentThread::getGroup()]()
             {
-                setThreadName("DiskCopier");
-                from_disk.copyFile(from_path, to_disk, fs::path(to_path) / fileName(from_path), settings);
+                try
+                {
+                    SCOPE_EXIT_SAFE(if (thread_group) CurrentThread::detachFromGroupIfNotDetached(););
+
+                    if (thread_group)
+                        CurrentThread::attachToGroup(thread_group);
+
+                    from_disk.copyFile(from_path, to_disk, fs::path(to_path) / fileName(from_path), settings);
+                    promise->set_value();
+                }
+                catch (...)
+                {
+                    promise->set_exception(std::current_exception());
+                }
             });
 
-        results.push_back(std::move(result));
+        results.push_back(std::move(future));
     }
     else
     {
@@ -104,13 +118,12 @@ void asyncCopy(IDisk & from_disk, String from_path, IDisk & to_disk, String to_p
         }
 
         for (auto it = from_disk.iterateDirectory(from_path); it->isValid(); it->next())
-            asyncCopy(from_disk, it->path(), to_disk, dest, exec, results, true, settings);
+            asyncCopy(from_disk, it->path(), to_disk, dest, pool, results, true, settings);
     }
 }
 
 void IDisk::copyThroughBuffers(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path, bool copy_root_dir)
 {
-    auto & exec = to_disk->getExecutor();
     ResultsCollector results;
 
     WriteSettings settings;
@@ -118,17 +131,12 @@ void IDisk::copyThroughBuffers(const String & from_path, const std::shared_ptr<I
     /// Avoid high memory usage. See test_s3_zero_copy_ttl/test.py::test_move_and_s3_memory_usage
     settings.s3_allow_parallel_part_upload = false;
 
-    asyncCopy(*this, from_path, *to_disk, to_path, exec, results, copy_root_dir, settings);
+    asyncCopy(*this, from_path, *to_disk, to_path, copying_thread_pool, results, copy_root_dir, settings);
 
     for (auto & result : results)
         result.wait();
     for (auto & result : results)
-        result.get();
-}
-
-void IDisk::copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path)
-{
-    copyThroughBuffers(from_path, to_disk, to_path, true);
+        result.get();   /// May rethrow an exception
 }
 
 
@@ -137,7 +145,7 @@ void IDisk::copyDirectoryContent(const String & from_dir, const std::shared_ptr<
     if (!to_disk->exists(to_dir))
         to_disk->createDirectories(to_dir);
 
-    copyThroughBuffers(from_dir, to_disk, to_dir, false);
+    copyThroughBuffers(from_dir, to_disk, to_dir, /* copy_root_dir */ false);
 }
 
 void IDisk::truncateFile(const String &, size_t)
@@ -233,4 +241,9 @@ catch (Exception & e)
     throw;
 }
 
+void IDisk::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr /*context*/, const String & config_prefix, const DisksMap & /*map*/)
+{
+    copying_thread_pool.setMaxThreads(config.getInt(config_prefix + ".thread_pool_size", 16));
+}
+
 }

src/Disks/IDisk.h

@@ -6,7 +6,6 @@
 #include <base/types.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/Exception.h>
-#include <Disks/Executor.h>
 #include <Disks/DiskType.h>
 #include <IO/ReadSettings.h>
 #include <IO/WriteSettings.h>
@@ -35,6 +34,12 @@ namespace Poco
     }
 }
 
+namespace CurrentMetrics
+{
+    extern const Metric IDiskCopierThreads;
+    extern const Metric IDiskCopierThreadsActive;
+}
+
 namespace DB
 {
 
@@ -110,9 +115,15 @@ class IDisk : public Space
 {
 public:
     /// Default constructor.
-    explicit IDisk(const String & name_, std::shared_ptr<Executor> executor_ = std::make_shared<SyncExecutor>())
+    IDisk(const String & name_, const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
         : name(name_)
-        , executor(executor_)
+        , copying_thread_pool(CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, config.getUInt(config_prefix + ".thread_pool_size", 16))
+    {
+    }
+
+    explicit IDisk(const String & name_)
+        : name(name_)
+        , copying_thread_pool(CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, 16)
     {
     }
 
@@ -181,9 +192,6 @@ public:
     /// If a file with `to_path` path already exists, it will be replaced.
     virtual void replaceFile(const String & from_path, const String & to_path) = 0;
 
-    /// Recursively copy data containing at `from_path` to `to_path` located at `to_disk`.
-    virtual void copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path);
-
     /// Recursively copy files from from_dir to to_dir. Create to_dir if not exists.
     virtual void copyDirectoryContent(const String & from_dir, const std::shared_ptr<IDisk> & to_disk, const String & to_dir);
 
@@ -379,7 +387,7 @@ public:
     virtual SyncGuardPtr getDirectorySyncGuard(const String & path) const;
 
     /// Applies new settings for disk in runtime.
-    virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextPtr, const String &, const DisksMap &) {}
+    virtual void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map);
 
     /// Quite leaky abstraction. Some disks can use additional disk to store
     /// some parts of metadata. In general case we have only one disk itself and
@@ -459,9 +467,6 @@ protected:
 
     const String name;
 
-    /// Returns executor to perform asynchronous operations.
-    virtual Executor & getExecutor() { return *executor; }
-
     /// Base implementation of the function copy().
     /// It just opens two files, reads data by portions from the first file, and writes it to the second one.
     /// A derived class may override copy() to provide a faster implementation.
@@ -470,7 +475,7 @@ protected:
     virtual void checkAccessImpl(const String & path);
 
 private:
-    std::shared_ptr<Executor> executor;
+    ThreadPool copying_thread_pool;
     bool is_custom_disk = false;
 
     /// Check access to the disk.

src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp

@@ -74,19 +74,22 @@ CachedOnDiskReadBufferFromFile::CachedOnDiskReadBufferFromFile(
 }
 
 void CachedOnDiskReadBufferFromFile::appendFilesystemCacheLog(
-    const FileSegment::Range & file_segment_range, CachedOnDiskReadBufferFromFile::ReadType type)
+    const FileSegment & file_segment, CachedOnDiskReadBufferFromFile::ReadType type)
 {
     if (!cache_log)
         return;
 
+    const auto range = file_segment.range();
     FilesystemCacheLogElement elem
     {
         .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()),
         .query_id = query_id,
         .source_file_path = source_file_path,
-        .file_segment_range = { file_segment_range.left, file_segment_range.right },
+        .file_segment_range = { range.left, range.right },
         .requested_range = { first_offset, read_until_position },
-        .file_segment_size = file_segment_range.size(),
+        .file_segment_key = file_segment.key().toString(),
+        .file_segment_offset = file_segment.offset(),
+        .file_segment_size = range.size(),
         .read_from_cache_attempted = true,
         .read_buffer_id = current_buffer_id,
         .profile_counters = std::make_shared<ProfileEvents::Counters::Snapshot>(
@@ -495,7 +498,7 @@ bool CachedOnDiskReadBufferFromFile::completeFileSegmentAndGetNext()
     auto completed_range = current_file_segment->range();
 
     if (cache_log)
-        appendFilesystemCacheLog(completed_range, read_type);
+        appendFilesystemCacheLog(*current_file_segment, read_type);
 
     chassert(file_offset_of_buffer_end > completed_range.right);
 
@@ -518,7 +521,7 @@ CachedOnDiskReadBufferFromFile::~CachedOnDiskReadBufferFromFile()
 {
     if (cache_log && file_segments && !file_segments->empty())
     {
-        appendFilesystemCacheLog(file_segments->front().range(), read_type);
+        appendFilesystemCacheLog(file_segments->front(), read_type);
     }
 }
 

src/Disks/IO/CachedOnDiskReadBufferFromFile.h

@@ -90,7 +90,7 @@ private:
 
     bool completeFileSegmentAndGetNext();
 
-    void appendFilesystemCacheLog(const FileSegment::Range & file_segment_range, ReadType read_type);
+    void appendFilesystemCacheLog(const FileSegment & file_segment, ReadType read_type);
 
     bool writeCache(char * data, size_t size, size_t offset, FileSegment & file_segment);
 

src/Disks/IO/ReadBufferFromRemoteFSGather.cpp

@@ -90,6 +90,8 @@ void ReadBufferFromRemoteFSGather::appendUncachedReadInfo()
         .source_file_path = current_object.remote_path,
         .file_segment_range = { 0, current_object.bytes_size },
         .cache_type = FilesystemCacheLogElement::CacheType::READ_FROM_FS_BYPASSING_CACHE,
+        .file_segment_key = {},
+        .file_segment_offset = {},
         .file_segment_size = current_object.bytes_size,
         .read_from_cache_attempted = false,
     };