Merge branch 'master' into multiStringAllPositions-non-const-needle

.github/workflows/master.yml

@@ -2902,7 +2902,7 @@ jobs:
 #############################################################################################
 #################################### PERFORMANCE TESTS ######################################
 #############################################################################################
-  PerformanceComparison0:
+  PerformanceComparisonX86-0:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -2940,7 +2940,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison1:
+  PerformanceComparisonX86-1:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -2978,7 +2978,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison2:
+  PerformanceComparisonX86-2:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3016,7 +3016,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison3:
+  PerformanceComparisonX86-3:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3096,10 +3096,10 @@ jobs:
       - IntegrationTestsTsan1
       - IntegrationTestsTsan2
       - IntegrationTestsTsan3
-      - PerformanceComparison0
-      - PerformanceComparison1
-      - PerformanceComparison2
-      - PerformanceComparison3
+      - PerformanceComparisonX86-0
+      - PerformanceComparisonX86-1
+      - PerformanceComparisonX86-2
+      - PerformanceComparisonX86-3
       - CompatibilityCheck
       - ASTFuzzerTestDebug
       - ASTFuzzerTestAsan

.github/workflows/pull_request.yml

@@ -3118,7 +3118,7 @@ jobs:
 #############################################################################################
 #################################### PERFORMANCE TESTS ######################################
 #############################################################################################
-  PerformanceComparison0:
+  PerformanceComparisonX86-0:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3156,7 +3156,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison1:
+  PerformanceComparisonX86-1:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3194,7 +3194,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison2:
+  PerformanceComparisonX86-2:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3232,7 +3232,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison3:
+  PerformanceComparisonX86-3:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3270,7 +3270,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch0:
+  PerformanceComparisonAarch-0:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3308,7 +3308,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch1:
+  PerformanceComparisonAarch-1:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3346,7 +3346,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch2:
+  PerformanceComparisonAarch-2:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3384,7 +3384,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch3:
+  PerformanceComparisonAarch-3:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3481,14 +3481,14 @@ jobs:
       - IntegrationTestsTsan1
       - IntegrationTestsTsan2
       - IntegrationTestsTsan3
-      - PerformanceComparison0
-      - PerformanceComparison1
-      - PerformanceComparison2
-      - PerformanceComparison3
-      - PerformanceComparisonAarch0
-      - PerformanceComparisonAarch1
-      - PerformanceComparisonAarch2
-      - PerformanceComparisonAarch3
+      - PerformanceComparisonX86-0
+      - PerformanceComparisonX86-1
+      - PerformanceComparisonX86-2
+      - PerformanceComparisonX86-3
+      - PerformanceComparisonAarch-0
+      - PerformanceComparisonAarch-1
+      - PerformanceComparisonAarch-2
+      - PerformanceComparisonAarch-3
       - UnitTestsAsan
       - UnitTestsTsan
       - UnitTestsMsan

CMakeLists.txt

@@ -335,6 +335,22 @@ if (COMPILER_GCC OR COMPILER_CLANG)
     set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
 endif ()
 
+if (ARCH_AMD64)
+    # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
+    # which makes benchmark results more stable.
+    set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
+    if (COMPILER_GCC)
+        # gcc is in assembler, need to add "-Wa," prefix
+        set(BRANCHES_WITHIN_32B_BOUNDARIES "-Wa,${BRANCHES_WITHIN_32B_BOUNDARIES}")
+    endif()
+
+    include(CheckCXXCompilerFlag)
+    check_cxx_compiler_flag("${BRANCHES_WITHIN_32B_BOUNDARIES}" HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
+    if (HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
+        set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
+    endif()
+endif()
+
 if (COMPILER_GCC)
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines")
 endif ()

base/base/JSON.cpp

@@ -12,7 +12,14 @@
 #define JSON_MAX_DEPTH 100
 
 
+#ifdef __clang__
+#  pragma clang diagnostic push
+#  pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec"
+#endif
 POCO_IMPLEMENT_EXCEPTION(JSONException, Poco::Exception, "JSONException") // NOLINT(cert-err60-cpp, modernize-use-noexcept, hicpp-use-noexcept)
+#ifdef __clang__
+#  pragma clang diagnostic pop
+#endif
 
 
 /// Прочитать беззнаковое целое в простом формате из не-0-terminated строки.

base/base/JSON.h

@@ -38,8 +38,14 @@
   */
 
 
+#ifdef __clang__
+#  pragma clang diagnostic push
+#  pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec"
+#endif
 POCO_DECLARE_EXCEPTION(Foundation_API, JSONException, Poco::Exception)
-
+#ifdef __clang__
+#  pragma clang diagnostic pop
+#endif
 
 class JSON
 {

cmake/warnings.cmake

@@ -20,16 +20,12 @@ if (COMPILER_CLANG)
     # We want to get everything out of the compiler for code quality.
     add_warning(everything)
     add_warning(pedantic)
-    no_warning(vla-extension)
     no_warning(zero-length-array)
-    no_warning(c11-extensions)
-    no_warning(unused-command-line-argument)
     no_warning(c++98-compat-pedantic)
     no_warning(c++98-compat)
     no_warning(c99-extensions)
     no_warning(conversion)
     no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
-    no_warning(deprecated-dynamic-exception-spec)
     no_warning(disabled-macro-expansion)
     no_warning(documentation-unknown-command)
     no_warning(double-promotion)
@@ -38,11 +34,7 @@ if (COMPILER_CLANG)
     no_warning(global-constructors)
     no_warning(missing-prototypes)
     no_warning(missing-variable-declarations)
-    no_warning(nested-anon-types)
-    no_warning(packed)
     no_warning(padded)
-    no_warning(return-std-move-in-c++11) # clang 7+
-    no_warning(shift-sign-overflow)
     no_warning(sign-conversion)
     no_warning(switch-enum)
     no_warning(undefined-func-template)

contrib/thrift

@@ -1 +1 @@
-Subproject commit 010ccf0a0c7023fea0f6bf4e4078ebdff7e61982
+Subproject commit 2a93df80f27739ccabb5b885cb12a8dc7595ecdf

contrib/thrift-cmake/CMakeLists.txt

@@ -15,7 +15,6 @@ set(thriftcpp_SOURCES
         "${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.cpp"
         "${LIBRARY_DIR}/src/thrift/concurrency/ThreadManager.cpp"
         "${LIBRARY_DIR}/src/thrift/concurrency/TimerManager.cpp"
-        "${LIBRARY_DIR}/src/thrift/concurrency/Util.cpp"
         "${LIBRARY_DIR}/src/thrift/processor/PeekProcessor.cpp"
         "${LIBRARY_DIR}/src/thrift/protocol/TBase64Utils.cpp"
         "${LIBRARY_DIR}/src/thrift/protocol/TDebugProtocol.cpp"
@@ -33,6 +32,8 @@ set(thriftcpp_SOURCES
         "${LIBRARY_DIR}/src/thrift/transport/TServerSocket.cpp"
         "${LIBRARY_DIR}/src/thrift/transport/TTransportUtils.cpp"
         "${LIBRARY_DIR}/src/thrift/transport/TBufferTransports.cpp"
+        "${LIBRARY_DIR}/src/thrift/transport/SocketCommon.cpp"
+        "${LIBRARY_DIR}/src/thrift/transport/TWebSocketServer.cpp"
         "${LIBRARY_DIR}/src/thrift/server/TConnectedClient.cpp"
         "${LIBRARY_DIR}/src/thrift/server/TServerFramework.cpp"
         "${LIBRARY_DIR}/src/thrift/server/TSimpleServer.cpp"
@@ -92,4 +93,4 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 add_library(_thrift ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
 add_library(ch_contrib::thrift ALIAS _thrift)
 target_include_directories(_thrift SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src" ${CMAKE_CURRENT_BINARY_DIR})
-target_link_libraries (_thrift PUBLIC boost::headers_only)
+target_link_libraries (_thrift PUBLIC OpenSSL::SSL boost::headers_only)

docker/test/fasttest/Dockerfile

@@ -71,6 +71,7 @@ RUN apt-get update \
         software-properties-common \
         tzdata \
         unixodbc \
+        file \
        --yes --no-install-recommends
 
 RUN pip3 install numpy scipy pandas Jinja2

docker/test/stateless/Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update -y \
             unixodbc \
             wget \
             zstd \
+            file \
     && apt-get clean
 
 

docker/test/stress/run.sh

@@ -353,6 +353,9 @@ else
 
     # Error messages (we should ignore some errors)
     # FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.")
+    # FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64")
+    # FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
+    #       Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
     echo "Check for Error messages in server log:"
     zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
                -e "Code: 236. DB::Exception: Cancelled mutating parts" \
@@ -376,6 +379,9 @@ else
                -e "found in queue and some source parts for it was lost" \
                -e "is lost forever." \
                -e "Unknown index: idx." \
+               -e "Cannot parse string 'Hello' as UInt64" \
+               -e "} <Error> TCPHandler: Code:" \
+               -e "} <Error> executeQuery: Code:" \
         /var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
         && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
         || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv

docs/en/operations/system-tables/part_log.md

@@ -19,6 +19,10 @@ The `system.part_log` table contains the following columns:
     -   `REGULAR_MERGE` — Some regular merge.
     -   `TTL_DELETE_MERGE` — Cleaning up expired data.
     -   `TTL_RECOMPRESS_MERGE` — Recompressing data part with the.
+-   `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Merge algorithm for the event with type `MERGE_PARTS`. Can have one of the following values:
+    -   `UNDECIDED`
+    -   `HORIZONTAL`
+    -   `VERTICAL`
 -   `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
 -   `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
 -   `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision.
@@ -52,6 +56,7 @@ Row 1:
 query_id:                      983ad9c7-28d5-4ae1-844e-603116b7de31
 event_type:                    NewPart
 merge_reason:                  NotAMerge
+merge_algorithm:               Undecided
 event_date:                    2021-02-02
 event_time:                    2021-02-02 11:14:28
 event_time_microseconds:                    2021-02-02 11:14:28.861919

docs/en/sql-reference/functions/array-functions.md

@@ -1715,3 +1715,7 @@ Return value type is always [Float64](../../sql-reference/data-types/float.md).
 │ 6   │ Float64                                                                                  │
 └─────┴──────────────────────────────────────────────────────────────────────────────────────────┘
 ```
+
+## Distance functions
+
+All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).

docs/en/sql-reference/functions/distance-functions.md

@@ -0,0 +1,502 @@
+# Distance functions
+
+## L1Norm
+
+Calculates the sum of absolute values of a vector.
+
+**Syntax**
+
+```sql
+L1Norm(vector)
+```
+
+Alias: `normL1`.
+
+**Arguments**
+
+-   `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance.
+
+Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
+
+**Examples**
+
+Query:
+
+```sql
+SELECT L1Norm((1, 2));
+```
+
+Result:
+
+```text
+┌─L1Norm((1, 2))─┐
+│              3 │
+└────────────────┘
+```
+
+## L2Norm
+
+Calculates the square root of the sum of the squares of the vector values.
+
+**Syntax**
+
+```sql
+L2Norm(vector)
+```
+
+Alias: `normL2`.
+
+**Arguments**
+
+-   `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance).
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L2Norm((1, 2));
+```
+
+Result:
+
+```text
+┌───L2Norm((1, 2))─┐
+│ 2.23606797749979 │
+└──────────────────┘
+```
+
+## LinfNorm
+
+Calculates the maximum of absolute values of a vector.
+
+**Syntax**
+
+```sql
+LinfNorm(vector)
+```
+
+Alias: `normLinf`.
+
+**Arguments**
+
+-   `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   Linf-norm or the maximum absolute value.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LinfNorm((1, -2));
+```
+
+Result:
+
+```text
+┌─LinfNorm((1, -2))─┐
+│                 2 │
+└───────────────────┘
+```
+
+## LpNorm
+
+Calculates the root of `p`-th power of the sum of the absolute values of a vector in the power of `p`.
+
+**Syntax**
+
+```sql
+LpNorm(vector, p)
+```
+
+Alias: `normLp`.
+
+**Arguments**
+
+-   `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `p` — The power. Possible values: real number in `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
+
+**Returned value**
+
+-   [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LpNorm((1, -2), 2);
+```
+
+Result:
+
+```text
+┌─LpNorm((1, -2), 2)─┐
+│   2.23606797749979 │
+└────────────────────┘
+```
+
+## L1Distance
+
+Calculates the distance between two points (the values of the vectors are the coordinates) in `L1` space (1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance)).
+
+**Syntax**
+
+```sql
+L1Distance(vector1, vector2)
+```
+
+Alias: `distanceL1`.
+
+**Arguments**
+
+-   `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   1-norm distance.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L1Distance((1, 2), (2, 3));
+```
+
+Result:
+
+```text
+┌─L1Distance((1, 2), (2, 3))─┐
+│                          2 │
+└────────────────────────────┘
+```
+
+## L2Distance
+
+Calculates the distance between two points (the values of the vectors are the coordinates) in Euclidean space ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
+
+**Syntax**
+
+```sql
+L2Distance(vector1, vector2)
+```
+
+Alias: `distanceL2`.
+
+**Arguments**
+
+-   `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   2-norm distance.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L2Distance((1, 2), (2, 3));
+```
+
+Result:
+
+```text
+┌─L2Distance((1, 2), (2, 3))─┐
+│         1.4142135623730951 │
+└────────────────────────────┘
+```
+
+## LinfDistance
+
+Calculates the distance between two points (the values of the vectors are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
+
+**Syntax**
+
+```sql
+LinfDistance(vector1, vector2)
+```
+
+Alias: `distanceLinf`.
+
+**Arguments**
+
+-   `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `vector1` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+-   Infinity-norm distance.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LinfDistance((1, 2), (2, 3));
+```
+
+Result:
+
+```text
+┌─LinfDistance((1, 2), (2, 3))─┐
+│                            1 │
+└──────────────────────────────┘
+```
+
+## LpDistance
+
+Calculates the distance between two points (the values of the vectors are the coordinates) in `Lp` space ([p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
+
+**Syntax**
+
+```sql
+LpDistance(vector1, vector2, p)
+```
+
+Alias: `distanceLp`.
+
+**Arguments**
+
+-   `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+-   `p` — The power. Possible values: real number from `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
+
+**Returned value**
+
+-   p-norm distance.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LpDistance((1, 2), (2, 3), 3);
+```
+
+Result:
+
+```text
+┌─LpDistance((1, 2), (2, 3), 3)─┐
+│            1.2599210498948732 │
+└───────────────────────────────┘
+```
+
+
+## L1Normalize
+
+Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L1` space ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)).
+
+**Syntax**
+
+```sql
+L1Normalize(tuple)
+```
+
+Alias: `normalizeL1`.
+
+**Arguments**
+
+-   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
+
+**Returned value**
+
+-   Unit vector.
+
+Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L1Normalize((1, 2));
+```
+
+Result:
+
+```text
+┌─L1Normalize((1, 2))─────────────────────┐
+│ (0.3333333333333333,0.6666666666666666) │
+└─────────────────────────────────────────┘
+```
+
+## L2Normalize
+
+Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in Euclidean space (using [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
+
+**Syntax**
+
+```sql
+L2Normalize(tuple)
+```
+
+Alias: `normalizeL1`.
+
+**Arguments**
+
+-   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
+
+**Returned value**
+
+-   Unit vector.
+
+Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L2Normalize((3, 4));
+```
+
+Result:
+
+```text
+┌─L2Normalize((3, 4))─┐
+│ (0.6,0.8)           │
+└─────────────────────┘
+```
+
+## LinfNormalize
+
+Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L_{inf}` space (using [maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
+
+**Syntax**
+
+```sql
+LinfNormalize(tuple)
+```
+
+Alias: `normalizeLinf `.
+
+**Arguments**
+
+-   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
+
+**Returned value**
+
+-   Unit vector.
+
+Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LinfNormalize((3, 4));
+```
+
+Result:
+
+```text
+┌─LinfNormalize((3, 4))─┐
+│ (0.75,1)              │
+└───────────────────────┘
+```
+
+## LpNormalize
+
+Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `Lp` space (using [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
+
+**Syntax**
+
+```sql
+LpNormalize(tuple, p)
+```
+
+Alias: `normalizeLp `.
+
+**Arguments**
+
+-   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
+-   `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
+
+**Returned value**
+
+-   Unit vector.
+
+Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT LpNormalize((3, 4),5);
+```
+
+Result:
+
+```text
+┌─LpNormalize((3, 4), 5)──────────────────┐
+│ (0.7187302630182624,0.9583070173576831) │
+└─────────────────────────────────────────┘
+```
+
+## cosineDistance
+
+Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors.
+
+**Syntax**
+
+```sql
+cosineDistance(tuple1, tuple2)
+```
+
+**Arguments**
+
+-   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
+-   `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
+
+**Returned value**
+
+-   Cosine of the angle between two vectors substracted from one.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT cosineDistance((1, 2), (2, 3));
+```
+
+Result:
+
+```text
+┌─cosineDistance((1, 2), (2, 3))─┐
+│           0.007722123286332261 │
+└────────────────────────────────┘
+```

docs/en/sql-reference/functions/string-replace-functions.md

@@ -93,3 +93,35 @@ Predefined characters: `\0`, `\\`, `|`, `(`, `)`, `^`, `$`, `.`, `[`, `]`, `?`,
 This implementation slightly differs from re2::RE2::QuoteMeta. It escapes zero byte as `\0` instead of `\x00` and it escapes only required characters.
 For more information, see the link: [RE2](https://github.com/google/re2/blob/master/re2/re2.cc#L473)
 
+
+## translate(s, from, to)
+
+The function replaces characters in the string ‘s’ in accordance with one-to-one character mapping defined by ‘from’ and ‘to’ strings. ‘from’ and ‘to’ must be ASCII strings of the same size. Non-ASCII characters in the original string are not modified.
+
+Example:
+
+``` sql
+SELECT translate('Hello, World!', 'delor', 'DELOR') AS res
+```
+
+``` text
+┌─res───────────┐
+│ HELLO, WORLD! │
+└───────────────┘
+```
+
+## translateUTF8(string, from, to)
+
+Similar to previous function, but works with UTF-8 arguments. ‘from’ and ‘to’ must be valid UTF-8 strings of the same size.
+
+Example:
+
+``` sql
+SELECT translateUTF8('Hélló, Wórld¡', 'óé¡', 'oe!') AS res
+```
+
+``` text
+┌─res───────────┐
+│ Hello, World! │
+└───────────────┘
+```

docs/en/sql-reference/functions/string-search-functions.md

@@ -374,7 +374,7 @@ The same as `multiMatchAny`, but returns any index that matches the haystack.
 
 ## multiMatchAllIndices(haystack, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
 
-The same as `multiMatchAny`, but returns the array of all indicies that match the haystack in any order.
+The same as `multiMatchAny`, but returns the array of all indices that match the haystack in any order.
 
 ## multiFuzzyMatchAny(haystack, distance, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
 

docs/en/sql-reference/functions/tuple-functions.md

@@ -559,502 +559,7 @@ Result:
 └────────────────────────────┘
 ```
 
-## L1Norm
 
-Calculates the sum of absolute values of a tuple.
+## Distance functions
 
-**Syntax**
-
-```sql
-L1Norm(tuple)
-```
-
-Alias: `normL1`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance.
-
-Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L1Norm((1, 2));
-```
-
-Result:
-
-```text
-┌─L1Norm((1, 2))─┐
-│              3 │
-└────────────────┘
-```
-
-## L2Norm
-
-Calculates the square root of the sum of the squares of the tuple values.
-
-**Syntax**
-
-```sql
-L2Norm(tuple)
-```
-
-Alias: `normL2`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance).
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L2Norm((1, 2));
-```
-
-Result:
-
-```text
-┌───L2Norm((1, 2))─┐
-│ 2.23606797749979 │
-└──────────────────┘
-```
-
-## LinfNorm
-
-Calculates the maximum of absolute values of a tuple.
-
-**Syntax**
-
-```sql
-LinfNorm(tuple)
-```
-
-Alias: `normLinf`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Linf-norm or the maximum absolute value.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LinfNorm((1, -2));
-```
-
-Result:
-
-```text
-┌─LinfNorm((1, -2))─┐
-│                 2 │
-└───────────────────┘
-```
-
-## LpNorm
-
-Calculates the root of `p`-th power of the sum of the absolute values of a tuple in the power of `p`.
-
-**Syntax**
-
-```sql
-LpNorm(tuple, p)
-```
-
-Alias: `normLp`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
--   `p` — The power. Possible values: real number in `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
-
-**Returned value**
-
--   [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LpNorm((1, -2), 2);
-```
-
-Result:
-
-```text
-┌─LpNorm((1, -2), 2)─┐
-│   2.23606797749979 │
-└────────────────────┘
-```
-
-## L1Distance
-
-Calculates the distance between two points (the values of the tuples are the coordinates) in `L1` space (1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance)).
-
-**Syntax**
-
-```sql
-L1Distance(tuple1, tuple2)
-```
-
-Alias: `distanceL1`.
-
-**Arguments**
-
--   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   1-norm distance.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L1Distance((1, 2), (2, 3));
-```
-
-Result:
-
-```text
-┌─L1Distance((1, 2), (2, 3))─┐
-│                          2 │
-└────────────────────────────┘
-```
-
-## L2Distance
-
-Calculates the distance between two points (the values of the tuples are the coordinates) in Euclidean space ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
-
-**Syntax**
-
-```sql
-L2Distance(tuple1, tuple2)
-```
-
-Alias: `distanceL2`.
-
-**Arguments**
-
--   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   2-norm distance.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L2Distance((1, 2), (2, 3));
-```
-
-Result:
-
-```text
-┌─L2Distance((1, 2), (2, 3))─┐
-│         1.4142135623730951 │
-└────────────────────────────┘
-```
-
-## LinfDistance
-
-Calculates the distance between two points (the values of the tuples are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
-
-**Syntax**
-
-```sql
-LinfDistance(tuple1, tuple2)
-```
-
-Alias: `distanceLinf`.
-
-**Arguments**
-
--   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Infinity-norm distance.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LinfDistance((1, 2), (2, 3));
-```
-
-Result:
-
-```text
-┌─LinfDistance((1, 2), (2, 3))─┐
-│                            1 │
-└──────────────────────────────┘
-```
-
-## LpDistance
-
-Calculates the distance between two points (the values of the tuples are the coordinates) in `Lp` space ([p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
-
-**Syntax**
-
-```sql
-LpDistance(tuple1, tuple2, p)
-```
-
-Alias: `distanceLp`.
-
-**Arguments**
-
--   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `p` — The power. Possible values: real number from `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
-
-**Returned value**
-
--   p-norm distance.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LpDistance((1, 2), (2, 3), 3);
-```
-
-Result:
-
-```text
-┌─LpDistance((1, 2), (2, 3), 3)─┐
-│            1.2599210498948732 │
-└───────────────────────────────┘
-```
-
-## L1Normalize
-
-Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L1` space ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)).
-
-**Syntax**
-
-```sql
-L1Normalize(tuple)
-```
-
-Alias: `normalizeL1`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Unit vector.
-
-Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L1Normalize((1, 2));
-```
-
-Result:
-
-```text
-┌─L1Normalize((1, 2))─────────────────────┐
-│ (0.3333333333333333,0.6666666666666666) │
-└─────────────────────────────────────────┘
-```
-
-## L2Normalize
-
-Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in Euclidean space (using [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
-
-**Syntax**
-
-```sql
-L2Normalize(tuple)
-```
-
-Alias: `normalizeL1`.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Unit vector.
-
-Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT L2Normalize((3, 4));
-```
-
-Result:
-
-```text
-┌─L2Normalize((3, 4))─┐
-│ (0.6,0.8)           │
-└─────────────────────┘
-```
-
-## LinfNormalize
-
-Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L_{inf}` space (using [maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
-
-**Syntax**
-
-```sql
-LinfNormalize(tuple)
-```
-
-Alias: `normalizeLinf `.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Unit vector.
-
-Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LinfNormalize((3, 4));
-```
-
-Result:
-
-```text
-┌─LinfNormalize((3, 4))─┐
-│ (0.75,1)              │
-└───────────────────────┘
-```
-
-## LpNormalize
-
-Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `Lp` space (using [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
-
-**Syntax**
-
-```sql
-LpNormalize(tuple, p)
-```
-
-Alias: `normalizeLp `.
-
-**Arguments**
-
--   `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
--   `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
-
-**Returned value**
-
--   Unit vector.
-
-Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT LpNormalize((3, 4),5);
-```
-
-Result:
-
-```text
-┌─LpNormalize((3, 4), 5)──────────────────┐
-│ (0.7187302630182624,0.9583070173576831) │
-└─────────────────────────────────────────┘
-```
-
-## cosineDistance
-
-Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors.
-
-**Syntax**
-
-```sql
-cosineDistance(tuple1, tuple2)
-```
-
-**Arguments**
-
--   `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
--   `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
-
-**Returned value**
-
--   Cosine of the angle between two vectors substracted from one.
-
-Type: [Float](../../sql-reference/data-types/float.md).
-
-**Example**
-
-Query:
-
-```sql
-SELECT cosineDistance((1, 2), (2, 3));
-```
-
-Result:
-
-```text
-┌─cosineDistance((1, 2), (2, 3))─┐
-│           0.007722123286332261 │
-└────────────────────────────────┘
-```
+All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).

docs/en/sql-reference/table-functions/null.md

@@ -5,7 +5,7 @@ sidebar_label: null function
 
 # null
 
-Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately droped right after the query execution. The function is used for the convenience of test writing and demonstrations.
+Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately dropped right after the query execution. The function is used for the convenience of test writing and demonstrations.
 
 **Syntax**
 
@@ -40,4 +40,3 @@ See also:
 
 -   [Null table engine](../../engines/table-engines/special/null.md)
 
-[Original article](https://clickhouse.com/docs/en/sql-reference/table-functions/null/) <!--hide-->

docs/en/sql-reference/table-functions/remote.md

@@ -5,11 +5,11 @@ sidebar_label: remote
 
 # remote, remoteSecure
 
-Allows to access remote servers without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. `remoteSecure` - same as `remote` but with a secured connection.
+Allows accessing remote servers, including migration of data, without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. `remoteSecure` - same as `remote` but with a secured connection.
 
 Both functions can be used in `SELECT` and `INSERT` queries.
 
-**Syntax**
+## Syntax
 
 ``` sql
 remote('addresses_expr', db, table[, 'user'[, 'password'], sharding_key])
@@ -18,7 +18,7 @@ remoteSecure('addresses_expr', db, table[, 'user'[, 'password'], sharding_key])
 remoteSecure('addresses_expr', db.table[, 'user'[, 'password'], sharding_key])
 ```
 
-**Parameters**
+## Parameters
 
 - `addresses_expr` — An expression that generates addresses of remote servers. This may be just one server address. The server address is `host:port`, or just `host`.
 
@@ -36,28 +36,31 @@ remoteSecure('addresses_expr', db.table[, 'user'[, 'password'], sharding_key])
 - `password` — User password. If the password is not specified, an empty password is used. Type: [String](../../sql-reference/data-types/string.md).
 - `sharding_key` — Sharding key to support distributing data across nodes. For example: `insert into remote('127.0.0.1:9000,127.0.0.2', db, table, 'default', rand())`. Type: [UInt32](../../sql-reference/data-types/int-uint.md).
 
-**Returned value**
+## Returned value
 
 The dataset from remote servers.
 
-**Usage**
+## Usage
 
-Using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function.
+Unless you are migrating data from one system to another, using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function.
 
 The `remote` table function can be useful in the following cases:
 
+- Migrating data from one system to another
 - Accessing a specific server for data comparison, debugging, and testing.
 - Queries between various ClickHouse clusters for research purposes.
 - Infrequent distributed requests that are made manually.
 - Distributed requests where the set of servers is re-defined each time.
 
-**Adresses**
+### Adresses
 
 ``` text
 example01-01-1
+example01-01-1:9440
 example01-01-1:9000
 localhost
 127.0.0.1
+[::]:9440
 [::]:9000
 [2a02:6b8:0:1111::11]:9000
 ```
@@ -68,15 +71,15 @@ Multiple addresses can be comma-separated. In this case, ClickHouse will use dis
 example01-01-1,example01-02-1
 ```
 
-**Examples**
+## Examples
 
-Selecting data from a remote server:
+### Selecting data from a remote server:
 
 ``` sql
 SELECT * FROM remote('127.0.0.1', db.remote_engine_table) LIMIT 3;
 ```
 
-Inserting data from a remote server into a table:
+### Inserting data from a remote server into a table:
 
 ``` sql
 CREATE TABLE remote_table (name String, value UInt32) ENGINE=Memory;
@@ -84,7 +87,65 @@ INSERT INTO FUNCTION remote('127.0.0.1', currentDatabase(), 'remote_table') VALU
 SELECT * FROM remote_table;
 ```
 
-## Globs in Addresses {globs-in-addresses}
+### Migration of tables from one system to another:
+This example uses one table from a sample dataset.  The database is `imdb`, and the table is `actors`. 
+
+#### On the source ClickHouse system (the system that currently hosts the data)
+- Verify the source database and table name (`imdb.actors`)  
+  ```sql
+  show databases
+  ```
+
+  ```sql
+  show tables in imdb
+  ```
+
+- Get the CREATE TABLE statement from the source:
+  ```
+  select create_table_query
+  from system.tables
+  where database = 'imdb' and table = 'actors'
+  ```
+
+  Response
+  ```sql
+  CREATE TABLE imdb.actors (`id` UInt32,
+                            `first_name` String,
+                            `last_name` String,
+                            `gender` FixedString(1))
+                  ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
+                  ORDER BY (id, first_name, last_name, gender)
+                  SETTINGS index_granularity = 8192
+  ```
+
+#### On the destination ClickHouse system:
+
+- Create the destination database:
+  ```sql
+  CREATE DATABASE imdb
+  ```
+
+- Using the CREATE TABLE statement from the source, create the destination:
+  ```sql
+  CREATE TABLE imdb.actors (`id` UInt32,
+                            `first_name` String,
+                            `last_name` String,
+                            `gender` FixedString(1))
+                  ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
+                  ORDER BY (id, first_name, last_name, gender)
+                  SETTINGS index_granularity = 8192
+  ```
+
+#### Back on the source deployment:
+
+Insert into the new database and table created on the remote system.  You will need the host, port, username, password, destination database, and destination table.
+```sql
+INSERT INTO FUNCTION
+remoteSecure('remote.clickhouse.cloud:9440', 'imdb.actors', 'USER', 'PASSWORD', rand())
+SELECT * from imdb.actors
+```
+
+## Globs in Addresses {#globs-in-addresses}
 
 Patterns in curly brackets `{ }` are used to generate a set of shards and to specify replicas. If there are multiple pairs of curly brackets, then the direct product of the corresponding sets is generated.
 The following pattern types are supported.

docs/ru/engines/table-engines/mergetree-family/mergetree.md

@@ -335,7 +335,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234
 
     Поддерживаемые типы данных: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`.
 
-    Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem).
+    Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions#hasany), [hasAll](../../../sql-reference/functions/array-functions#hasall).
 
 **Примеры**
 

docs/ru/operations/system-tables/part_log.md

@@ -14,6 +14,15 @@
     -   `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
     -   `MUTATE_PART` — изменение куска.
     -   `MOVE_PART` — перемещение куска между дисками.
+-   `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — Причина события с типом `MERGE_PARTS`. Может принимать одно из следующих значений:
+    -   `NOT_A_MERGE` — событие имеет тип иной, чем `MERGE_PARTS`.
+    -   `REGULAR_MERGE` — обычное слияние.
+    -   `TTL_DELETE_MERGE` — очистка истекших данных.
+    -   `TTL_RECOMPRESS_MERGE` — переупаковка куска.
+-   `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Алгоритм слияния для события с типом `MERGE_PARTS`. Может принимать одно из следующих значений:
+    -   `UNDECIDED`
+    -   `HORIZONTAL`
+    -   `VERTICAL`
 -   `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата события.
 -   `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время события.
 -   `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время события с точностью до микросекунд.
@@ -46,6 +55,8 @@ Row 1:
 ──────
 query_id:                      983ad9c7-28d5-4ae1-844e-603116b7de31
 event_type:                    NewPart
+merge_reason:                  NotAMerge
+merge_algorithm:               Undecided
 event_date:                    2021-02-02
 event_time:                    2021-02-02 11:14:28
 event_time_microseconds:                    2021-02-02 11:14:28.861919

docs/ru/sql-reference/functions/string-replace-functions.md

@@ -83,3 +83,34 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res
 └─────────────────────┘
 ```
 
+## translate(s, from, to)
+
+Данная функция заменяет символы в строке ‘s’ в соответствии с поэлементным отображением определяемым строками ‘from’ и ‘to’. ‘from’ и ‘to’ должны быть корректными ASCII строками одного размера. Не ASCII символы в оригинальной строке не изменяются.
+
+Example:
+
+``` sql
+SELECT translate('Hello, World!', 'delor', 'DELOR') AS res
+```
+
+``` text
+┌─res───────────┐
+│ HELLO, WORLD! │
+└───────────────┘
+```
+
+## translateUTF8(string, from, to)
+
+Аналогично предыдущей функции, но работает со строками, состоящими из UTF-8 символов. ‘from’ и ‘to’ должны быть корректными UTF-8 строками одного размера.
+
+Example:
+
+``` sql
+SELECT translateUTF8('Hélló, Wórld¡', 'óé¡', 'oe!') AS res
+```
+
+``` text
+┌─res───────────┐
+│ Hello, World! │
+└───────────────┘
+```

programs/copier/ZooKeeperStaff.h

@@ -20,7 +20,7 @@ public:
 
     bool operator<=(const WrappingUInt32 & other) const
     {
-        const UInt32 HALF = 1 << 31;
+        const UInt32 HALF = static_cast<UInt32>(1) << 31;
         return (value <= other.value && other.value - value < HALF)
                || (value > other.value && value - other.value > HALF);
     }

src/Access/AccessControl.cpp

@@ -101,7 +101,7 @@ public:
         registered_prefixes = prefixes_;
     }
 
-    bool isSettingNameAllowed(const std::string_view & setting_name) const
+    bool isSettingNameAllowed(std::string_view setting_name) const
     {
         if (Settings::hasBuiltin(setting_name))
             return true;
@@ -116,7 +116,7 @@ public:
         return false;
     }
 
-    void checkSettingNameIsAllowed(const std::string_view & setting_name) const
+    void checkSettingNameIsAllowed(std::string_view setting_name) const
     {
         if (isSettingNameAllowed(setting_name))
             return;

src/Access/AccessRights.cpp

@@ -252,7 +252,7 @@ public:
     }
 
     template <typename ... Args>
-    void grant(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
+    void grant(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
     {
         auto & child = getChild(name);
         child.grant(flags_, subnames...);
@@ -279,7 +279,7 @@ public:
     }
 
     template <typename... Args>
-    void revoke(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
+    void revoke(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
     {
         auto & child = getChild(name);
 
@@ -306,7 +306,7 @@ public:
     }
 
     template <typename... Args>
-    bool isGranted(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames) const
+    bool isGranted(const AccessFlags & flags_, std::string_view name, const Args &... subnames) const
     {
         AccessFlags flags_to_check = flags_ - min_flags_with_children;
         if (!flags_to_check)
@@ -415,7 +415,7 @@ private:
     AccessFlags getAllGrantableFlags() const { return ::DB::getAllGrantableFlags(level); }
     AccessFlags getChildAllGrantableFlags() const { return ::DB::getAllGrantableFlags(static_cast<Level>(level + 1)); }
 
-    Node * tryGetChild(const std::string_view & name) const
+    Node * tryGetChild(std::string_view name) const
     {
         if (!children)
             return nullptr;
@@ -425,7 +425,7 @@ private:
         return &it->second;
     }
 
-    Node & getChild(const std::string_view & name)
+    Node & getChild(std::string_view name)
     {
         auto * child = tryGetChild(name);
         if (child)
@@ -819,20 +819,20 @@ void AccessRights::grantImpl(const AccessRightsElements & elements)
 }
 
 void AccessRights::grant(const AccessFlags & flags) { grantImpl<false>(flags); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl<false>(flags, database); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<false>(flags, database, table); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<false>(flags, database, table, column); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database) { grantImpl<false>(flags, database); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<false>(flags, database, table); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<false>(flags, database, table, column); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
 void AccessRights::grant(const AccessRightsElement & element) { grantImpl<false>(element); }
 void AccessRights::grant(const AccessRightsElements & elements) { grantImpl<false>(elements); }
 
 void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl<true>(flags); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database) { grantImpl<true>(flags, database); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<true>(flags, database, table); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<true>(flags, database, table, column); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database) { grantImpl<true>(flags, database); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<true>(flags, database, table); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<true>(flags, database, table, column); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
 void AccessRights::grantWithGrantOption(const AccessRightsElement & element) { grantImpl<true>(element); }
 void AccessRights::grantWithGrantOption(const AccessRightsElements & elements) { grantImpl<true>(elements); }
 
@@ -892,20 +892,20 @@ void AccessRights::revokeImpl(const AccessRightsElements & elements)
 }
 
 void AccessRights::revoke(const AccessFlags & flags) { revokeImpl<false>(flags); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl<false>(flags, database); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<false>(flags, database, table); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<false>(flags, database, table, column); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database) { revokeImpl<false>(flags, database); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<false>(flags, database, table); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<false>(flags, database, table, column); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
 void AccessRights::revoke(const AccessRightsElement & element) { revokeImpl<false>(element); }
 void AccessRights::revoke(const AccessRightsElements & elements) { revokeImpl<false>(elements); }
 
 void AccessRights::revokeGrantOption(const AccessFlags & flags) { revokeImpl<true>(flags); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database) { revokeImpl<true>(flags, database); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<true>(flags, database, table); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<true>(flags, database, table, column); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database) { revokeImpl<true>(flags, database); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<true>(flags, database, table); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<true>(flags, database, table, column); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
 void AccessRights::revokeGrantOption(const AccessRightsElement & element) { revokeImpl<true>(element); }
 void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { revokeImpl<true>(elements); }
 
@@ -984,20 +984,20 @@ bool AccessRights::isGrantedImpl(const AccessRightsElements & elements) const
 }
 
 bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl<false>(flags); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<false>(flags, database); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<false>(flags, database, table); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<false>(flags, database, table, column); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<false>(flags, database); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<false>(flags, database, table); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<false>(flags, database, table, column); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
 bool AccessRights::isGranted(const AccessRightsElement & element) const { return isGrantedImpl<false>(element); }
 bool AccessRights::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl<false>(elements); }
 
 bool AccessRights::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl<true>(flags); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<true>(flags, database); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<true>(flags, database, table); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<true>(flags, database, table, column); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<true>(flags, database); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<true>(flags, database, table); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<true>(flags, database, table, column); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
 bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl<true>(element); }
 bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl<true>(elements); }
 

src/Access/AccessRights.h

@@ -39,59 +39,59 @@ public:
     /// Grants access on a specified database/table/column.
     /// Does nothing if the specified access has been already granted.
     void grant(const AccessFlags & flags);
-    void grant(const AccessFlags & flags, const std::string_view & database);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void grant(const AccessFlags & flags, std::string_view database);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void grant(const AccessRightsElement & element);
     void grant(const AccessRightsElements & elements);
 
     void grantWithGrantOption(const AccessFlags & flags);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void grantWithGrantOption(const AccessRightsElement & element);
     void grantWithGrantOption(const AccessRightsElements & elements);
 
     /// Revokes a specified access granted earlier on a specified database/table/column.
     /// For example, revoke(AccessType::ALL) revokes all grants at all, just like clear();
     void revoke(const AccessFlags & flags);
-    void revoke(const AccessFlags & flags, const std::string_view & database);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void revoke(const AccessFlags & flags, std::string_view database);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void revoke(const AccessRightsElement & element);
     void revoke(const AccessRightsElements & elements);
 
     void revokeGrantOption(const AccessFlags & flags);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void revokeGrantOption(const AccessRightsElement & element);
     void revokeGrantOption(const AccessRightsElements & elements);
 
     /// Whether a specified access granted.
     bool isGranted(const AccessFlags & flags) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool isGranted(const AccessRightsElement & element) const;
     bool isGranted(const AccessRightsElements & elements) const;
 
     bool hasGrantOption(const AccessFlags & flags) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool hasGrantOption(const AccessRightsElement & element) const;
     bool hasGrantOption(const AccessRightsElements & elements) const;
 
@@ -107,9 +107,9 @@ public:
         const AccessFlags & flags,
         const AccessFlags & min_flags_with_children,
         const AccessFlags & max_flags_with_children,
-        const std::string_view & database,
-        const std::string_view & table,
-        const std::string_view & column)>;
+        std::string_view database,
+        std::string_view table,
+        std::string_view column)>;
     void modifyFlags(const ModifyFlagsFunction & function);
     void modifyFlagsWithGrantOption(const ModifyFlagsFunction & function);
 

src/Access/Authentication.cpp

@@ -26,17 +26,17 @@ namespace
         return (Util::stringToDigest(password) == password_plaintext);
     }
 
-    bool checkPasswordDoubleSHA1(const std::string_view & password, const Digest & password_double_sha1)
+    bool checkPasswordDoubleSHA1(std::string_view password, const Digest & password_double_sha1)
     {
         return (Util::encodeDoubleSHA1(password) == password_double_sha1);
     }
 
-    bool checkPasswordSHA256(const std::string_view & password, const Digest & password_sha256, const String & salt)
+    bool checkPasswordSHA256(std::string_view password, const Digest & password_sha256, const String & salt)
     {
         return Util::encodeSHA256(String(password).append(salt)) == password_sha256;
     }
 
-    bool checkPasswordDoubleSHA1MySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_double_sha1)
+    bool checkPasswordDoubleSHA1MySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_double_sha1)
     {
         /// scrambled_password = SHA1(password) XOR SHA1(scramble <concat> SHA1(SHA1(password)))
 
@@ -61,7 +61,7 @@ namespace
         return calculated_password_double_sha1 == password_double_sha1;
     }
 
-    bool checkPasswordPlainTextMySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_plaintext)
+    bool checkPasswordPlainTextMySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_plaintext)
     {
         return checkPasswordDoubleSHA1MySQL(scramble, scrambled_password, Util::encodeDoubleSHA1(password_plaintext));
     }

src/Access/Common/AccessFlags.cpp

@@ -35,7 +35,7 @@ namespace
             return access_type_to_flags_mapping[static_cast<size_t>(type)];
         }
 
-        Flags keywordToFlags(const std::string_view & keyword) const
+        Flags keywordToFlags(std::string_view keyword) const
         {
             auto it = keyword_to_flags_map.find(keyword);
             if (it == keyword_to_flags_map.end())
@@ -142,14 +142,14 @@ namespace
             }
         };
 
-        static String replaceUnderscoreWithSpace(const std::string_view & str)
+        static String replaceUnderscoreWithSpace(std::string_view str)
         {
             String res{str};
             boost::replace_all(res, "_", " ");
             return res;
         }
 
-        static Strings splitAliases(const std::string_view & str)
+        static Strings splitAliases(std::string_view str)
         {
             Strings aliases;
             boost::split(aliases, str, boost::is_any_of(","));
@@ -160,10 +160,10 @@ namespace
 
         static void makeNode(
             AccessType access_type,
-            const std::string_view & name,
-            const std::string_view & aliases,
+            std::string_view name,
+            std::string_view aliases,
             NodeType node_type,
-            const std::string_view & parent_group_name,
+            std::string_view parent_group_name,
             std::unordered_map<std::string_view, Node *> & nodes,
             std::unordered_map<std::string_view, NodePtr> & owned_nodes,
             size_t & next_flag)
@@ -353,7 +353,7 @@ namespace
 
 
 AccessFlags::AccessFlags(AccessType type) : flags(Helper::instance().accessTypeToFlags(type)) {}
-AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
+AccessFlags::AccessFlags(std::string_view keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
 AccessFlags::AccessFlags(const std::vector<std::string_view> & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
 AccessFlags::AccessFlags(const Strings & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
 String AccessFlags::toString() const { return Helper::instance().flagsToString(flags); }

src/Access/Common/AccessFlags.h

@@ -21,7 +21,7 @@ public:
     AccessFlags() = default;
 
     /// Constructs from a string like "SELECT".
-    AccessFlags(const std::string_view & keyword); /// NOLINT
+    AccessFlags(std::string_view keyword); /// NOLINT
 
     /// Constructs from a list of strings like "SELECT, UPDATE, INSERT".
     AccessFlags(const std::vector<std::string_view> & keywords); /// NOLINT

src/Access/Common/AccessRightsElement.cpp

@@ -81,7 +81,7 @@ namespace
         }
 
         bool need_comma = false;
-        for (const std::string_view & keyword : keywords)
+        for (std::string_view keyword : keywords)
         {
             if (need_comma)
                 result.append(", ");
@@ -145,18 +145,18 @@ namespace
 }
 
 
-AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_)
+AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_)
     : access_flags(access_flags_), database(database_), any_database(false)
 {
 }
 
-AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_)
+AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_)
     : access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false)
 {
 }
 
 AccessRightsElement::AccessRightsElement(
-    AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_)
+    AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_)
     : access_flags(access_flags_)
     , database(database_)
     , table(table_)
@@ -169,8 +169,8 @@ AccessRightsElement::AccessRightsElement(
 
 AccessRightsElement::AccessRightsElement(
     AccessFlags access_flags_,
-    const std::string_view & database_,
-    const std::string_view & table_,
+    std::string_view database_,
+    std::string_view table_,
     const std::vector<std::string_view> & columns_)
     : access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false), any_column(false)
 {
@@ -180,7 +180,7 @@ AccessRightsElement::AccessRightsElement(
 }
 
 AccessRightsElement::AccessRightsElement(
-    AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_)
+    AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_)
     : access_flags(access_flags_)
     , database(database_)
     , table(table_)

src/Access/Common/AccessRightsElement.h

@@ -28,19 +28,19 @@ struct AccessRightsElement
 
     explicit AccessRightsElement(AccessFlags access_flags_) : access_flags(access_flags_) {}
 
-    AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_);
-    AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_);
+    AccessRightsElement(AccessFlags access_flags_, std::string_view database_);
+    AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_);
     AccessRightsElement(
-        AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_);
+        AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_);
 
     AccessRightsElement(
         AccessFlags access_flags_,
-        const std::string_view & database_,
-        const std::string_view & table_,
+        std::string_view database_,
+        std::string_view table_,
         const std::vector<std::string_view> & columns_);
 
     AccessRightsElement(
-        AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_);
+        AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_);
 
     bool empty() const { return !access_flags || (!any_column && columns.empty()); }
 

src/Access/Common/AccessType.cpp

@@ -35,7 +35,7 @@ namespace
 #undef ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING
         }
 
-        void addToMapping(AccessType type, const std::string_view & str)
+        void addToMapping(AccessType type, std::string_view str)
         {
             String str2{str};
             boost::replace_all(str2, "_", " ");

src/Access/Common/AuthenticationData.cpp

@@ -71,7 +71,7 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty
 }
 
 
-AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::string_view & text [[maybe_unused]])
+AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_view text [[maybe_unused]])
 {
 #if USE_SSL
     Digest hash;
@@ -86,7 +86,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::str
 }
 
 
-AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(const std::string_view & text)
+AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(std::string_view text)
 {
     Poco::SHA1Engine engine;
     engine.update(text.data(), text.size());

src/Access/Common/AuthenticationData.h

@@ -96,11 +96,11 @@ public:
 
     struct Util
     {
-        static Digest stringToDigest(const std::string_view & text) { return Digest(text.data(), text.data() + text.size()); }
-        static Digest encodeSHA256(const std::string_view & text);
-        static Digest encodeSHA1(const std::string_view & text);
+        static Digest stringToDigest(std::string_view text) { return Digest(text.data(), text.data() + text.size()); }
+        static Digest encodeSHA256(std::string_view text);
+        static Digest encodeSHA1(std::string_view text);
         static Digest encodeSHA1(const Digest & text) { return encodeSHA1(std::string_view{reinterpret_cast<const char *>(text.data()), text.size()}); }
-        static Digest encodeDoubleSHA1(const std::string_view & text) { return encodeSHA1(encodeSHA1(text)); }
+        static Digest encodeDoubleSHA1(std::string_view text) { return encodeSHA1(encodeSHA1(text)); }
         static Digest encodeDoubleSHA1(const Digest & text) { return encodeSHA1(encodeSHA1(text)); }
     };
 

src/Access/ContextAccess.cpp

@@ -46,7 +46,7 @@ namespace
 
     AccessRights addImplicitAccessRights(const AccessRights & access)
     {
-        auto modifier = [&](const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, const std::string_view & database, const std::string_view & table, const std::string_view & column) -> AccessFlags
+        auto modifier = [&](const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, std::string_view database, std::string_view table, std::string_view column) -> AccessFlags
         {
             size_t level = !database.empty() + !table.empty() + !column.empty();
             AccessFlags res = flags;
@@ -141,7 +141,7 @@ namespace
     std::string_view getDatabase() { return {}; }
 
     template <typename... OtherArgs>
-    std::string_view getDatabase(const std::string_view & arg1, const OtherArgs &...) { return arg1; }
+    std::string_view getDatabase(std::string_view arg1, const OtherArgs &...) { return arg1; }
 }
 
 
@@ -519,7 +519,7 @@ bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const
 }
 
 template <bool throw_if_denied, bool grant_option, typename... Args>
-bool ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const
+bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const
 {
     return checkAccessImplHelper<throw_if_denied, grant_option>(flags, database.empty() ? params.current_database : database, args...);
 }
@@ -564,38 +564,38 @@ bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
 }
 
 bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl<false, false>(flags); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, false>(flags, database); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, false>(flags, database, table); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(flags, database); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(flags, database, table); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
 bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl<false, false>(element); }
 bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(elements); }
 
 bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return checkAccessImpl<false, true>(flags); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, true>(flags, database); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, true>(flags, database, table); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(flags, database); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(flags, database, table); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
 bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return checkAccessImpl<false, true>(element); }
 bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(elements); }
 
 void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<true, false>(flags); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, false>(flags, database); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, false>(flags, database, table); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, false>(flags, database, table, column); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(flags, database); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(flags, database, table); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(flags, database, table, column); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
 void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<true, false>(element); }
 void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<true, false>(elements); }
 
 void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<true, true>(flags); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, true>(flags, database); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, true>(flags, database, table); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, true>(flags, database, table, column); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(flags, database); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(flags, database, table); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(flags, database, table, column); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
 void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<true, true>(element); }
 void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true, true>(elements); }
 

src/Access/ContextAccess.h

@@ -101,40 +101,40 @@ public:
     /// Checks if a specified access is granted, and throws an exception if not.
     /// Empty database means the current database.
     void checkAccess(const AccessFlags & flags) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     void checkAccess(const AccessRightsElement & element) const;
     void checkAccess(const AccessRightsElements & elements) const;
 
     void checkGrantOption(const AccessFlags & flags) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     void checkGrantOption(const AccessRightsElement & element) const;
     void checkGrantOption(const AccessRightsElements & elements) const;
 
     /// Checks if a specified access is granted, and returns false if not.
     /// Empty database means the current database.
     bool isGranted(const AccessFlags & flags) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool isGranted(const AccessRightsElement & element) const;
     bool isGranted(const AccessRightsElements & elements) const;
 
     bool hasGrantOption(const AccessFlags & flags) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool hasGrantOption(const AccessRightsElement & element) const;
     bool hasGrantOption(const AccessRightsElements & elements) const;
 
@@ -180,7 +180,7 @@ private:
     bool checkAccessImpl(const AccessFlags & flags) const;
 
     template <bool throw_if_denied, bool grant_option, typename... Args>
-    bool checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const;
+    bool checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const;
 
     template <bool throw_if_denied, bool grant_option>
     bool checkAccessImpl(const AccessRightsElement & element) const;

src/Access/SettingsConstraints.cpp

@@ -36,12 +36,12 @@ void SettingsConstraints::clear()
 }
 
 
-void SettingsConstraints::setMinValue(const std::string_view & setting_name, const Field & min_value)
+void SettingsConstraints::setMinValue(std::string_view setting_name, const Field & min_value)
 {
     getConstraintRef(setting_name).min_value = Settings::castValueUtil(setting_name, min_value);
 }
 
-Field SettingsConstraints::getMinValue(const std::string_view & setting_name) const
+Field SettingsConstraints::getMinValue(std::string_view setting_name) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -51,12 +51,12 @@ Field SettingsConstraints::getMinValue(const std::string_view & setting_name) co
 }
 
 
-void SettingsConstraints::setMaxValue(const std::string_view & setting_name, const Field & max_value)
+void SettingsConstraints::setMaxValue(std::string_view setting_name, const Field & max_value)
 {
     getConstraintRef(setting_name).max_value = Settings::castValueUtil(setting_name, max_value);
 }
 
-Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) const
+Field SettingsConstraints::getMaxValue(std::string_view setting_name) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -66,12 +66,12 @@ Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) co
 }
 
 
-void SettingsConstraints::setReadOnly(const std::string_view & setting_name, bool read_only)
+void SettingsConstraints::setReadOnly(std::string_view setting_name, bool read_only)
 {
     getConstraintRef(setting_name).read_only = read_only;
 }
 
-bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) const
+bool SettingsConstraints::isReadOnly(std::string_view setting_name) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -81,7 +81,7 @@ bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) cons
 }
 
 
-void SettingsConstraints::set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only)
+void SettingsConstraints::set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only)
 {
     auto & ref = getConstraintRef(setting_name);
     ref.min_value = Settings::castValueUtil(setting_name, min_value);
@@ -89,7 +89,7 @@ void SettingsConstraints::set(const std::string_view & setting_name, const Field
     ref.read_only = read_only;
 }
 
-void SettingsConstraints::get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const
+void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -318,7 +318,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh
 }
 
 
-SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const std::string_view & setting_name)
+SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(std::string_view setting_name)
 {
     auto it = constraints.find(setting_name);
     if (it == constraints.end())
@@ -331,7 +331,7 @@ SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const st
     return it->second;
 }
 
-const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(const std::string_view & setting_name) const
+const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(std::string_view setting_name) const
 {
     auto it = constraints.find(setting_name);
     if (it == constraints.end())

src/Access/SettingsConstraints.h

@@ -61,17 +61,17 @@ public:
     void clear();
     bool empty() const { return constraints.empty(); }
 
-    void setMinValue(const std::string_view & setting_name, const Field & min_value);
-    Field getMinValue(const std::string_view & setting_name) const;
+    void setMinValue(std::string_view setting_name, const Field & min_value);
+    Field getMinValue(std::string_view setting_name) const;
 
-    void setMaxValue(const std::string_view & setting_name, const Field & max_value);
-    Field getMaxValue(const std::string_view & setting_name) const;
+    void setMaxValue(std::string_view setting_name, const Field & max_value);
+    Field getMaxValue(std::string_view setting_name) const;
 
-    void setReadOnly(const std::string_view & setting_name, bool read_only);
-    bool isReadOnly(const std::string_view & setting_name) const;
+    void setReadOnly(std::string_view setting_name, bool read_only);
+    bool isReadOnly(std::string_view setting_name) const;
 
-    void set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only);
-    void get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const;
+    void set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only);
+    void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const;
 
     void merge(const SettingsConstraints & other);
 
@@ -105,8 +105,8 @@ private:
     };
     bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const;
 
-    Constraint & getConstraintRef(const std::string_view & setting_name);
-    const Constraint * tryGetConstraint(const std::string_view & setting_name) const;
+    Constraint & getConstraintRef(std::string_view setting_name);
+    const Constraint * tryGetConstraint(std::string_view setting_name) const;
 
     std::unordered_map<std::string_view, Constraint> constraints;
     const AccessControl * access_control = nullptr;

src/CMakeLists.txt

@@ -401,11 +401,11 @@ dbms_target_link_libraries (
         clickhouse_parsers
         ch_contrib::lz4
         Poco::JSON
-        Poco::MongoDB
         string_utils
     PUBLIC
         boost::system
         clickhouse_common_io
+        Poco::MongoDB
 )
 
 if (TARGET ch::mysqlxx)

src/Client/ClientBase.cpp

@@ -601,6 +601,7 @@ void ClientBase::initLogsOutputStream()
     {
         WriteBuffer * wb = out_logs_buf.get();
 
+        bool color_logs = false;
         if (!out_logs_buf)
         {
             if (server_logs_file.empty())
@@ -608,11 +609,13 @@ void ClientBase::initLogsOutputStream()
                 /// Use stderr by default
                 out_logs_buf = std::make_unique<WriteBufferFromFileDescriptor>(STDERR_FILENO);
                 wb = out_logs_buf.get();
+                color_logs = stderr_is_a_tty;
             }
             else if (server_logs_file == "-")
             {
                 /// Use stdout if --server_logs_file=- specified
                 wb = &std_out;
+                color_logs = stdout_is_a_tty;
             }
             else
             {
@@ -622,7 +625,7 @@ void ClientBase::initLogsOutputStream()
             }
         }
 
-        logs_out_stream = std::make_unique<InternalTextLogs>(*wb, stdout_is_a_tty);
+        logs_out_stream = std::make_unique<InternalTextLogs>(*wb, color_logs);
     }
 }
 
@@ -2155,6 +2158,7 @@ void ClientBase::init(int argc, char ** argv)
 
     stdin_is_a_tty = isatty(STDIN_FILENO);
     stdout_is_a_tty = isatty(STDOUT_FILENO);
+    stderr_is_a_tty = isatty(STDERR_FILENO);
     terminal_width = getTerminalWidth();
 
     Arguments common_arguments{""}; /// 0th argument is ignored.

src/Client/ClientBase.h

@@ -173,6 +173,7 @@ protected:
 
     bool stdin_is_a_tty = false; /// stdin is a terminal.
     bool stdout_is_a_tty = false; /// stdout is a terminal.
+    bool stderr_is_a_tty = false; /// stderr is a terminal.
     uint64_t terminal_width = 0;
 
     ServerConnectionPtr connection;

src/Common/FileSegment.h

@@ -4,7 +4,7 @@
 #include <Common/IFileCache.h>
 #include <Core/Types.h>
 #include <IO/WriteBufferFromFile.h>
-#include <IO/SeekableReadBuffer.h>
+#include <IO/ReadBufferFromFileBase.h>
 #include <list>
 
 namespace Poco { class Logger; }
@@ -33,7 +33,7 @@ friend class FileSegmentRangeWriter;
 
 public:
     using Key = IFileCache::Key;
-    using RemoteFileReaderPtr = std::shared_ptr<SeekableReadBuffer>;
+    using RemoteFileReaderPtr = std::shared_ptr<ReadBufferFromFileBase>;
     using LocalCacheWriterPtr = std::unique_ptr<WriteBufferFromFile>;
 
     enum class State

src/Common/JSONParsers/DummyJSONParser.h

@@ -84,7 +84,7 @@ struct DummyJSONParser
         static Iterator begin() { return {}; }
         static Iterator end() { return {}; }
         static size_t size() { return 0; }
-        bool find(const std::string_view &, Element &) const { return false; } /// NOLINT
+        bool find(std::string_view, Element &) const { return false; } /// NOLINT
 
 #if 0
         /// Optional: Provides access to an object's element by index.
@@ -93,7 +93,7 @@ struct DummyJSONParser
     };
 
     /// Parses a JSON document, returns the reference to its root element if succeeded.
-    bool parse(const std::string_view &, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
+    bool parse(std::string_view, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
 
 #if 0
     /// Optional: Allocates memory to parse JSON documents faster.

src/Common/JSONParsers/RapidJSONParser.h

@@ -98,7 +98,7 @@ struct RapidJSONParser
         ALWAYS_INLINE Iterator end() const { return ptr->MemberEnd(); }
         ALWAYS_INLINE size_t size() const { return ptr->MemberCount(); }
 
-        bool find(const std::string_view & key, Element & result) const
+        bool find(std::string_view key, Element & result) const
         {
             auto it = ptr->FindMember(rapidjson::StringRef(key.data(), key.length()));
             if (it == ptr->MemberEnd())
@@ -122,7 +122,7 @@ struct RapidJSONParser
     };
 
     /// Parses a JSON document, returns the reference to its root element if succeeded.
-    bool parse(const std::string_view & json, Element & result)
+    bool parse(std::string_view json, Element & result)
     {
         rapidjson::MemoryStream ms(json.data(), json.size());
         rapidjson::EncodedInputStream<rapidjson::UTF8<>, rapidjson::MemoryStream> is(ms);

src/Common/JSONParsers/SimdJSONParser.h

@@ -105,7 +105,7 @@ struct SimdJSONParser
         ALWAYS_INLINE Iterator end() const { return object.end(); }
         ALWAYS_INLINE size_t size() const { return object.size(); }
 
-        bool find(const std::string_view & key, Element & result) const
+        bool find(std::string_view key, Element & result) const
         {
             auto x = object.at_key(key);
             if (x.error())
@@ -131,7 +131,7 @@ struct SimdJSONParser
     };
 
     /// Parses a JSON document, returns the reference to its root element if succeeded.
-    bool parse(const std::string_view & json, Element & result)
+    bool parse(std::string_view json, Element & result)
     {
         auto document = parser.parse(json.data(), json.size());
         if (document.error())

src/Common/OpenSSLHelpers.cpp

@@ -10,7 +10,7 @@ namespace DB
 {
 #pragma GCC diagnostic warning "-Wold-style-cast"
 
-std::string encodeSHA256(const std::string_view & text)
+std::string encodeSHA256(std::string_view text)
 {
     return encodeSHA256(text.data(), text.size());
 }
@@ -21,7 +21,7 @@ std::string encodeSHA256(const void * text, size_t size)
     encodeSHA256(text, size, reinterpret_cast<unsigned char *>(out.data()));
     return out;
 }
-void encodeSHA256(const std::string_view & text, unsigned char * out)
+void encodeSHA256(std::string_view text, unsigned char * out)
 {
     encodeSHA256(text.data(), text.size(), out);
 }

src/Common/OpenSSLHelpers.h

@@ -10,10 +10,10 @@ namespace DB
 {
 
 /// Encodes `text` and returns it.
-std::string encodeSHA256(const std::string_view & text);
+std::string encodeSHA256(std::string_view text);
 std::string encodeSHA256(const void * text, size_t size);
 /// `out` must be at least 32 bytes long.
-void encodeSHA256(const std::string_view & text, unsigned char * out);
+void encodeSHA256(std::string_view text, unsigned char * out);
 void encodeSHA256(const void * text, size_t size, unsigned char * out);
 
 /// Returns concatenation of error strings for all errors that OpenSSL has recorded, emptying the error queue.

src/Common/SettingsChanges.cpp

@@ -4,7 +4,7 @@ namespace DB
 {
 namespace
 {
-    SettingChange * find(SettingsChanges & changes, const std::string_view & name)
+    SettingChange * find(SettingsChanges & changes, std::string_view name)
     {
         auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
         if (it == changes.end())
@@ -12,7 +12,7 @@ namespace
         return &*it;
     }
 
-    const SettingChange * find(const SettingsChanges & changes, const std::string_view & name)
+    const SettingChange * find(const SettingsChanges & changes, std::string_view name)
     {
         auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
         if (it == changes.end())
@@ -21,7 +21,7 @@ namespace
     }
 }
 
-bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) const
+bool SettingsChanges::tryGet(std::string_view name, Field & out_value) const
 {
     const auto * change = find(*this, name);
     if (!change)
@@ -30,7 +30,7 @@ bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) c
     return true;
 }
 
-const Field * SettingsChanges::tryGet(const std::string_view & name) const
+const Field * SettingsChanges::tryGet(std::string_view name) const
 {
     const auto * change = find(*this, name);
     if (!change)
@@ -38,7 +38,7 @@ const Field * SettingsChanges::tryGet(const std::string_view & name) const
     return &change->value;
 }
 
-Field * SettingsChanges::tryGet(const std::string_view & name)
+Field * SettingsChanges::tryGet(std::string_view name)
 {
     auto * change = find(*this, name);
     if (!change)

src/Common/SettingsChanges.h

@@ -14,8 +14,8 @@ struct SettingChange
     Field value;
 
     SettingChange() = default;
-    SettingChange(const std::string_view & name_, const Field & value_) : name(name_), value(value_) {}
-    SettingChange(const std::string_view & name_, Field && value_) : name(name_), value(std::move(value_)) {}
+    SettingChange(std::string_view name_, const Field & value_) : name(name_), value(value_) {}
+    SettingChange(std::string_view name_, Field && value_) : name(name_), value(std::move(value_)) {}
 
     friend bool operator ==(const SettingChange & lhs, const SettingChange & rhs) { return (lhs.name == rhs.name) && (lhs.value == rhs.value); }
     friend bool operator !=(const SettingChange & lhs, const SettingChange & rhs) { return !(lhs == rhs); }
@@ -27,9 +27,9 @@ class SettingsChanges : public std::vector<SettingChange>
 public:
     using std::vector<SettingChange>::vector;
 
-    bool tryGet(const std::string_view & name, Field & out_value) const;
-    const Field * tryGet(const std::string_view & name) const;
-    Field * tryGet(const std::string_view & name);
+    bool tryGet(std::string_view name, Field & out_value) const;
+    const Field * tryGet(std::string_view name) const;
+    Field * tryGet(std::string_view name);
 };
 
 }

src/Common/StringUtils/StringUtils.h

@@ -147,7 +147,7 @@ inline bool isPunctuationASCII(char c)
 }
 
 
-inline bool isValidIdentifier(const std::string_view & str)
+inline bool isValidIdentifier(std::string_view str)
 {
     return !str.empty()
         && isValidIdentifierBegin(str[0])

src/Common/TaskStatsInfoGetter.cpp

@@ -21,6 +21,7 @@
 
 #if defined(__clang__)
     #pragma clang diagnostic ignored "-Wgnu-anonymous-struct"
+    #pragma clang diagnostic ignored "-Wnested-anon-types"
 #endif
 
 /// Basic idea is motivated by "iotop" tool.

src/Common/ThreadPool.h

@@ -162,20 +162,19 @@ public:
 
     template <typename Function, typename... Args>
     explicit ThreadFromGlobalPool(Function && func, Args &&... args)
-        : state(std::make_shared<Poco::Event>())
-        , thread_id(std::make_shared<std::thread::id>())
+        : state(std::make_shared<State>())
     {
-        /// NOTE: If this will throw an exception, the destructor won't be called.
+        /// NOTE:
+        /// - If this will throw an exception, the destructor won't be called
+        /// - this pointer cannot be passed in the lambda, since after detach() it will not be valid
         GlobalThreadPool::instance().scheduleOrThrow([
-            thread_id = thread_id,
             state = state,
             func = std::forward<Function>(func),
             args = std::make_tuple(std::forward<Args>(args)...)]() mutable /// mutable is needed to destroy capture
         {
-            auto event = std::move(state);
-            SCOPE_EXIT(event->set());
+            SCOPE_EXIT(state->event.set());
 
-            thread_id = std::make_shared<std::thread::id>(std::this_thread::get_id());
+            state->thread_id = std::this_thread::get_id();
 
             /// This moves are needed to destroy function and arguments before exit.
             /// It will guarantee that after ThreadFromGlobalPool::join all captured params are destroyed.
@@ -196,31 +195,30 @@ public:
 
     ThreadFromGlobalPool & operator=(ThreadFromGlobalPool && rhs) noexcept
     {
-        if (joinable())
+        if (initialized())
             abort();
         state = std::move(rhs.state);
-        thread_id = std::move(rhs.thread_id);
         return *this;
     }
 
     ~ThreadFromGlobalPool()
     {
-        if (joinable())
+        if (initialized())
             abort();
     }
 
     void join()
     {
-        if (!joinable())
+        if (!initialized())
             abort();
 
-        state->wait();
+        state->event.wait();
         state.reset();
     }
 
     void detach()
     {
-        if (!joinable())
+        if (!initialized())
             abort();
         state.reset();
     }
@@ -230,15 +228,30 @@ public:
         if (!state)
             return false;
         /// Thread cannot join itself.
-        if (*thread_id == std::this_thread::get_id())
+        if (state->thread_id == std::this_thread::get_id())
             return false;
         return true;
     }
 
 private:
+    struct State
+    {
+        /// Should be atomic() because of possible concurrent access between
+        /// assignment and joinable() check.
+        std::atomic<std::thread::id> thread_id;
+
         /// The state used in this object and inside the thread job.
-    std::shared_ptr<Poco::Event> state;
-    std::shared_ptr<std::thread::id> thread_id;
+        Poco::Event event;
+    };
+    std::shared_ptr<State> state;
+
+    /// Internally initialized() should be used over joinable(),
+    /// since it is enough to know that the thread is initialized,
+    /// and ignore that fact that thread cannot join itself.
+    bool initialized() const
+    {
+        return static_cast<bool>(state);
+    }
 };
 
 

src/Common/Throttler.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <Common/Throttler_fwd.h>
+
 #include <mutex>
 #include <memory>
 #include <base/sleep.h>
@@ -57,7 +59,4 @@ private:
     std::shared_ptr<Throttler> parent;
 };
 
-
-using ThrottlerPtr = std::shared_ptr<Throttler>;
-
 }

src/Common/Throttler_fwd.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <memory>
+
+namespace DB
+{
+
+class Throttler;
+using ThrottlerPtr = std::shared_ptr<Throttler>;
+
+}

src/Common/filesystemHelpers.cpp

@@ -229,6 +229,35 @@ size_t getSizeFromFileDescriptor(int fd, const String & file_name)
     return buf.st_size;
 }
 
+int getINodeNumberFromPath(const String & path)
+{
+    struct stat file_stat;
+    if (stat(path.data(), &file_stat))
+    {
+        throwFromErrnoWithPath(
+            "Cannot execute stat for file " + path,
+            path,
+            ErrorCodes::CANNOT_STAT);
+    }
+    return file_stat.st_ino;
+}
+
+std::optional<size_t> tryGetSizeFromFilePath(const String & path)
+{
+    std::error_code ec;
+
+    size_t size = fs::file_size(path, ec);
+    if (!ec)
+        return size;
+
+    if (ec == std::errc::no_such_file_or_directory)
+        return std::nullopt;
+    if (ec == std::errc::operation_not_supported)
+        return std::nullopt;
+
+    throw fs::filesystem_error("Got unexpected error while getting file size", path, ec);
+}
+
 }
 
 

src/Common/filesystemHelpers.h

@@ -66,6 +66,12 @@ bool fileOrSymlinkPathStartsWith(const String & path, const String & prefix_path
 
 size_t getSizeFromFileDescriptor(int fd, const String & file_name = "");
 
+std::optional<size_t> tryGetSizeFromFilePath(const String & path);
+
+/// Get inode number for a file path.
+/// Will not work correctly if filesystem does not support inodes.
+int getINodeNumberFromPath(const String & path);
+
 }
 
 namespace FS

src/Common/mysqlxx/mysqlxx/Exception.h

@@ -12,8 +12,8 @@ struct Exception : public Poco::Exception
 {
     explicit Exception(const std::string & msg, int code = 0) : Poco::Exception(msg, code) {}
     int errnum() const { return code(); }
-    const char * name() const throw() override { return "mysqlxx::Exception"; }
-    const char * className() const throw() override { return "mysqlxx::Exception"; }
+    const char * name() const noexcept override { return "mysqlxx::Exception"; }
+    const char * className() const noexcept override { return "mysqlxx::Exception"; }
 };
 
 
@@ -21,8 +21,8 @@ struct Exception : public Poco::Exception
 struct ConnectionFailed : public Exception
 {
     explicit ConnectionFailed(const std::string & msg, int code = 0) : Exception(msg, code) {}
-    const char * name() const throw() override { return "mysqlxx::ConnectionFailed"; }
-    const char * className() const throw() override { return "mysqlxx::ConnectionFailed"; }
+    const char * name() const noexcept override { return "mysqlxx::ConnectionFailed"; }
+    const char * className() const noexcept override { return "mysqlxx::ConnectionFailed"; }
 };
 
 
@@ -30,8 +30,8 @@ struct ConnectionFailed : public Exception
 struct ConnectionLost : public Exception
 {
     explicit ConnectionLost(const std::string & msg, int code = 0) : Exception(msg, code) {}
-    const char * name() const throw() override { return "mysqlxx::ConnectionLost"; }
-    const char * className() const throw() override { return "mysqlxx::ConnectionLost"; }
+    const char * name() const noexcept override { return "mysqlxx::ConnectionLost"; }
+    const char * className() const noexcept override { return "mysqlxx::ConnectionLost"; }
 };
 
 
@@ -39,8 +39,8 @@ struct ConnectionLost : public Exception
 struct BadQuery : public Exception
 {
     explicit BadQuery(const std::string & msg, int code = 0) : Exception(msg, code) {}
-    const char * name() const throw() override { return "mysqlxx::BadQuery"; }
-    const char * className() const throw() override { return "mysqlxx::BadQuery"; }
+    const char * name() const noexcept override { return "mysqlxx::BadQuery"; }
+    const char * className() const noexcept override { return "mysqlxx::BadQuery"; }
 };
 
 
@@ -48,8 +48,8 @@ struct BadQuery : public Exception
 struct CannotParseValue : public Exception
 {
     explicit CannotParseValue(const std::string & msg, int code = 0) : Exception(msg, code) {}
-    const char * name() const throw() override { return "mysqlxx::CannotParseValue"; }
-    const char * className() const throw() override { return "mysqlxx::CannotParseValue"; }
+    const char * name() const noexcept override { return "mysqlxx::CannotParseValue"; }
+    const char * className() const noexcept override { return "mysqlxx::CannotParseValue"; }
 };
 
 

src/Common/parseGlobs.cpp

@@ -32,7 +32,7 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob
     }
     std::string escaped_with_globs = buf_for_escaping.str();
 
-    static const re2::RE2 enum_or_range(R"({([\d]+\.\.[\d]+|[^{}*,]+,[^{}*]*[^{}*,])})");    /// regexp for {expr1,expr2,expr3} or {M..N}, where M and N - non-negative integers, expr's should be without {}*,
+    static const re2::RE2 enum_or_range(R"({([\d]+\.\.[\d]+|[^{}*,]+,[^{}*]*[^{}*,])})");    /// regexp for {expr1,expr2,expr3} or {M..N}, where M and N - non-negative integers, expr's should be without "{", "}", "*" and ","
     re2::StringPiece input(escaped_with_globs);
     re2::StringPiece matched;
     std::ostringstream oss_for_replacing;       // STYLE_CHECK_ALLOW_STD_STRING_STREAM
@@ -50,16 +50,32 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob
             char point;
             ReadBufferFromString buf_range(buffer);
             buf_range >> range_begin >> point >> point >> range_end;
+
+            size_t range_begin_width = buffer.find('.');
+            size_t range_end_width = buffer.size() - buffer.find_last_of('.') - 1;
             bool leading_zeros = buffer[0] == '0';
-            size_t num_len = std::to_string(range_end).size();
+            size_t output_width = 0;
+
+            if (range_begin > range_end)    //Descending Sequence {20..15} {9..01}
+            {
+                std::swap(range_begin,range_end);
+                leading_zeros = buffer[buffer.find_last_of('.')+1]=='0';
+                std::swap(range_begin_width,range_end_width);
+            }
+            if (range_begin_width == 1 && leading_zeros)
+                output_width = 1;   ///Special Case: {0..10} {0..999}
+            else
+                output_width = std::max(range_begin_width, range_end_width);
+
             if (leading_zeros)
-                oss_for_replacing << std::setfill('0') << std::setw(num_len);
+                oss_for_replacing << std::setfill('0') << std::setw(output_width);
             oss_for_replacing << range_begin;
+
             for (size_t i = range_begin + 1; i <= range_end; ++i)
             {
                 oss_for_replacing << '|';
                 if (leading_zeros)
-                    oss_for_replacing << std::setfill('0') << std::setw(num_len);
+                    oss_for_replacing << std::setfill('0') << std::setw(output_width);
                 oss_for_replacing << i;
             }
         }

src/Common/tests/gtest_makeRegexpPatternFromGlobs.cpp

@@ -8,14 +8,36 @@ using namespace DB;
 
 TEST(Common, makeRegexpPatternFromGlobs)
 {
+
     EXPECT_EQ(makeRegexpPatternFromGlobs("?"), "[^/]");
     EXPECT_EQ(makeRegexpPatternFromGlobs("*"), "[^/]*");
     EXPECT_EQ(makeRegexpPatternFromGlobs("/?"), "/[^/]");
     EXPECT_EQ(makeRegexpPatternFromGlobs("/*"), "/[^/]*");
     EXPECT_EQ(makeRegexpPatternFromGlobs("*_{{a,b,c,d}}/?.csv"), "[^/]*_\\{(a|b|c|d)\\}/[^/]\\.csv");
-    EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..09}"), "f(1|2|3|4|5|6|7|8|9)");
-    EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..9}"), "f(1|2|3|4|5|6|7|8|9)");
-    EXPECT_EQ(makeRegexpPatternFromGlobs("f{0001..0000009}"), "f(1|2|3|4|5|6|7|8|9)");
+    /* Regex Parsing for {..} can have three possible cases
+       1) The left range width == the right range width
+       2) The left range width > the right range width
+       3) The left range width < the right range width
+    */
+    // Ascending Sequences
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{1..9}"), "f(1|2|3|4|5|6|7|8|9)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{0..10}"), "f(0|1|2|3|4|5|6|7|8|9|10)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{10..20}"), "f(10|11|12|13|14|15|16|17|18|19|20)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{00..10}"), "f(00|01|02|03|04|05|06|07|08|09|10)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{0001..0009}"), "f(0001|0002|0003|0004|0005|0006|0007|0008|0009)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..9}"), "f(01|02|03|04|05|06|07|08|09)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{000..9}"), "f(000|001|002|003|004|005|006|007|008|009)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{95..103}"), "f(95|96|97|98|99|100|101|102|103)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{99..109}"), "f(99|100|101|102|103|104|105|106|107|108|109)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{001..0009}"), "f(0001|0002|0003|0004|0005|0006|0007|0008|0009)");
+    // Descending Sequences
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{20..15}"), "f(15|16|17|18|19|20)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{200..199}"), "f(199|200)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{0009..0001}"), "f(0001|0002|0003|0004|0005|0006|0007|0008|0009)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{100..90}"), "f(90|91|92|93|94|95|96|97|98|99|100)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{103..95}"), "f(95|96|97|98|99|100|101|102|103)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{9..01}"), "f(01|02|03|04|05|06|07|08|09)");
+    EXPECT_EQ(makeRegexpPatternFromGlobs("f{9..000}"), "f(000|001|002|003|004|005|006|007|008|009)");
     EXPECT_EQ(makeRegexpPatternFromGlobs("f{1..2}{1..2}"), "f(1|2)(1|2)");
     EXPECT_EQ(makeRegexpPatternFromGlobs("f{1..1}{1..1}"), "f(1)(1)");
     EXPECT_EQ(makeRegexpPatternFromGlobs("f{0..0}{0..0}"), "f(0)(0)");

src/Compression/CompressionCodecEncrypted.cpp

@@ -131,7 +131,7 @@ std::string lastErrorString()
 /// This function get key and nonce and encrypt text with their help.
 /// If something went wrong (can't init context or can't encrypt data) it throws exception.
 /// It returns length of encrypted text.
-size_t encrypt(const std::string_view & plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce)
+size_t encrypt(std::string_view plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce)
 {
     /// Init context for encryption, using key.
     EVP_AEAD_CTX encrypt_ctx;
@@ -160,7 +160,7 @@ size_t encrypt(const std::string_view & plaintext, char * ciphertext_and_tag, En
 /// This function get key and nonce and encrypt text with their help.
 /// If something went wrong (can't init context or can't encrypt data) it throws exception.
 /// It returns length of encrypted text.
-size_t decrypt(const std::string_view & ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce)
+size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce)
 {
     /// Init context for decryption with given key.
     EVP_AEAD_CTX decrypt_ctx;

src/Core/BaseSettings.cpp

@@ -11,7 +11,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_SETTING;
 }
 
-void BaseSettingsHelpers::writeString(const std::string_view & str, WriteBuffer & out)
+void BaseSettingsHelpers::writeString(std::string_view str, WriteBuffer & out)
 {
     writeStringBinary(str, out);
 }
@@ -39,13 +39,13 @@ BaseSettingsHelpers::Flags BaseSettingsHelpers::readFlags(ReadBuffer & in)
 }
 
 
-void BaseSettingsHelpers::throwSettingNotFound(const std::string_view & name)
+void BaseSettingsHelpers::throwSettingNotFound(std::string_view name)
 {
     throw Exception("Unknown setting " + String{name}, ErrorCodes::UNKNOWN_SETTING);
 }
 
 
-void BaseSettingsHelpers::warningSettingNotFound(const std::string_view & name)
+void BaseSettingsHelpers::warningSettingNotFound(std::string_view name)
 {
     static auto * log = &Poco::Logger::get("Settings");
     LOG_WARNING(log, "Unknown setting {}, skipping", name);

src/Core/BaseSettings.h

@@ -45,16 +45,16 @@ class BaseSettings : public TTraits::Data
 public:
     using Traits = TTraits;
 
-    void set(const std::string_view & name, const Field & value);
-    Field get(const std::string_view & name) const;
+    void set(std::string_view name, const Field & value);
+    Field get(std::string_view name) const;
 
-    void setString(const std::string_view & name, const String & value);
-    String getString(const std::string_view & name) const;
+    void setString(std::string_view name, const String & value);
+    String getString(std::string_view name) const;
 
-    bool tryGet(const std::string_view & name, Field & value) const;
-    bool tryGetString(const std::string_view & name, String & value) const;
+    bool tryGet(std::string_view name, Field & value) const;
+    bool tryGetString(std::string_view name, String & value) const;
 
-    bool isChanged(const std::string_view & name) const;
+    bool isChanged(std::string_view name) const;
     SettingsChanges changes() const;
     void applyChange(const SettingChange & change);
     void applyChanges(const SettingsChanges & changes);
@@ -63,22 +63,22 @@ public:
     /// Resets all the settings to their default values.
     void resetToDefault();
 
-    bool has(const std::string_view & name) const { return hasBuiltin(name) || hasCustom(name); }
-    static bool hasBuiltin(const std::string_view & name);
-    bool hasCustom(const std::string_view & name) const;
+    bool has(std::string_view name) const { return hasBuiltin(name) || hasCustom(name); }
+    static bool hasBuiltin(std::string_view name);
+    bool hasCustom(std::string_view name) const;
 
-    const char * getTypeName(const std::string_view & name) const;
-    const char * getDescription(const std::string_view & name) const;
+    const char * getTypeName(std::string_view name) const;
+    const char * getDescription(std::string_view name) const;
 
     /// Checks if it's possible to assign a field to a specified value and throws an exception if not.
     /// This function doesn't change the fields, it performs check only.
-    static void checkCanSet(const std::string_view & name, const Field & value);
-    static void checkCanSetString(const std::string_view & name, const String & str);
+    static void checkCanSet(std::string_view name, const Field & value);
+    static void checkCanSetString(std::string_view name, const String & str);
 
     /// Conversions without changing the fields.
-    static Field castValueUtil(const std::string_view & name, const Field & value);
-    static String valueToStringUtil(const std::string_view & name, const Field & value);
-    static Field stringToValueUtil(const std::string_view & name, const String & str);
+    static Field castValueUtil(std::string_view name, const Field & value);
+    static String valueToStringUtil(std::string_view name, const Field & value);
+    static Field stringToValueUtil(std::string_view name, const String & str);
 
     void write(WriteBuffer & out, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT) const;
     void read(ReadBuffer & in, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT);
@@ -164,19 +164,19 @@ public:
     Iterator end() const { return allChanged().end(); }
 
 private:
-    SettingFieldCustom & getCustomSetting(const std::string_view & name);
-    const SettingFieldCustom & getCustomSetting(const std::string_view & name) const;
-    const SettingFieldCustom * tryGetCustomSetting(const std::string_view & name) const;
+    SettingFieldCustom & getCustomSetting(std::string_view name);
+    const SettingFieldCustom & getCustomSetting(std::string_view name) const;
+    const SettingFieldCustom * tryGetCustomSetting(std::string_view name) const;
 
     std::conditional_t<Traits::allow_custom_settings, CustomSettingMap, boost::blank> custom_settings_map;
 };
 
 struct BaseSettingsHelpers
 {
-    [[noreturn]] static void throwSettingNotFound(const std::string_view & name);
-    static void warningSettingNotFound(const std::string_view & name);
+    [[noreturn]] static void throwSettingNotFound(std::string_view name);
+    static void warningSettingNotFound(std::string_view name);
 
-    static void writeString(const std::string_view & str, WriteBuffer & out);
+    static void writeString(std::string_view str, WriteBuffer & out);
     static String readString(ReadBuffer & in);
 
     enum Flags : UInt64
@@ -190,7 +190,7 @@ struct BaseSettingsHelpers
 };
 
 template <typename TTraits>
-void BaseSettings<TTraits>::set(const std::string_view & name, const Field & value)
+void BaseSettings<TTraits>::set(std::string_view name, const Field & value)
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -200,7 +200,7 @@ void BaseSettings<TTraits>::set(const std::string_view & name, const Field & val
 }
 
 template <typename TTraits>
-Field BaseSettings<TTraits>::get(const std::string_view & name) const
+Field BaseSettings<TTraits>::get(std::string_view name) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -210,7 +210,7 @@ Field BaseSettings<TTraits>::get(const std::string_view & name) const
 }
 
 template <typename TTraits>
-void BaseSettings<TTraits>::setString(const std::string_view & name, const String & value)
+void BaseSettings<TTraits>::setString(std::string_view name, const String & value)
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -220,7 +220,7 @@ void BaseSettings<TTraits>::setString(const std::string_view & name, const Strin
 }
 
 template <typename TTraits>
-String BaseSettings<TTraits>::getString(const std::string_view & name) const
+String BaseSettings<TTraits>::getString(std::string_view name) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -230,7 +230,7 @@ String BaseSettings<TTraits>::getString(const std::string_view & name) const
 }
 
 template <typename TTraits>
-bool BaseSettings<TTraits>::tryGet(const std::string_view & name, Field & value) const
+bool BaseSettings<TTraits>::tryGet(std::string_view name, Field & value) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -247,7 +247,7 @@ bool BaseSettings<TTraits>::tryGet(const std::string_view & name, Field & value)
 }
 
 template <typename TTraits>
-bool BaseSettings<TTraits>::tryGetString(const std::string_view & name, String & value) const
+bool BaseSettings<TTraits>::tryGetString(std::string_view name, String & value) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -264,7 +264,7 @@ bool BaseSettings<TTraits>::tryGetString(const std::string_view & name, String &
 }
 
 template <typename TTraits>
-bool BaseSettings<TTraits>::isChanged(const std::string_view & name) const
+bool BaseSettings<TTraits>::isChanged(std::string_view name) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -316,20 +316,20 @@ void BaseSettings<TTraits>::resetToDefault()
 }
 
 template <typename TTraits>
-bool BaseSettings<TTraits>::hasBuiltin(const std::string_view & name)
+bool BaseSettings<TTraits>::hasBuiltin(std::string_view name)
 {
     const auto & accessor = Traits::Accessor::instance();
     return (accessor.find(name) != static_cast<size_t>(-1));
 }
 
 template <typename TTraits>
-bool BaseSettings<TTraits>::hasCustom(const std::string_view & name) const
+bool BaseSettings<TTraits>::hasCustom(std::string_view name) const
 {
     return tryGetCustomSetting(name);
 }
 
 template <typename TTraits>
-const char * BaseSettings<TTraits>::getTypeName(const std::string_view & name) const
+const char * BaseSettings<TTraits>::getTypeName(std::string_view name) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -341,7 +341,7 @@ const char * BaseSettings<TTraits>::getTypeName(const std::string_view & name) c
 }
 
 template <typename TTraits>
-const char * BaseSettings<TTraits>::getDescription(const std::string_view & name) const
+const char * BaseSettings<TTraits>::getDescription(std::string_view name) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -353,19 +353,19 @@ const char * BaseSettings<TTraits>::getDescription(const std::string_view & name
 }
 
 template <typename TTraits>
-void BaseSettings<TTraits>::checkCanSet(const std::string_view & name, const Field & value)
+void BaseSettings<TTraits>::checkCanSet(std::string_view name, const Field & value)
 {
     castValueUtil(name, value);
 }
 
 template <typename TTraits>
-void BaseSettings<TTraits>::checkCanSetString(const std::string_view & name, const String & str)
+void BaseSettings<TTraits>::checkCanSetString(std::string_view name, const String & str)
 {
     stringToValueUtil(name, str);
 }
 
 template <typename TTraits>
-Field BaseSettings<TTraits>::castValueUtil(const std::string_view & name, const Field & value)
+Field BaseSettings<TTraits>::castValueUtil(std::string_view name, const Field & value)
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -377,7 +377,7 @@ Field BaseSettings<TTraits>::castValueUtil(const std::string_view & name, const
 }
 
 template <typename TTraits>
-String BaseSettings<TTraits>::valueToStringUtil(const std::string_view & name, const Field & value)
+String BaseSettings<TTraits>::valueToStringUtil(std::string_view name, const Field & value)
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -389,7 +389,7 @@ String BaseSettings<TTraits>::valueToStringUtil(const std::string_view & name, c
 }
 
 template <typename TTraits>
-Field BaseSettings<TTraits>::stringToValueUtil(const std::string_view & name, const String & str)
+Field BaseSettings<TTraits>::stringToValueUtil(std::string_view name, const String & str)
 {
     try
     {
@@ -521,7 +521,7 @@ bool operator!=(const BaseSettings<TTraits> & left, const BaseSettings<TTraits>
 }
 
 template <typename TTraits>
-SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_view & name)
+SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(std::string_view name)
 {
     if constexpr (Traits::allow_custom_settings)
     {
@@ -537,7 +537,7 @@ SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_v
 }
 
 template <typename TTraits>
-const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_view & name) const
+const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(std::string_view name) const
 {
     if constexpr (Traits::allow_custom_settings)
     {
@@ -549,7 +549,7 @@ const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::st
 }
 
 template <typename TTraits>
-const SettingFieldCustom * BaseSettings<TTraits>::tryGetCustomSetting(const std::string_view & name) const
+const SettingFieldCustom * BaseSettings<TTraits>::tryGetCustomSetting(std::string_view name) const
 {
     if constexpr (Traits::allow_custom_settings)
     {
@@ -780,7 +780,7 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
         public: \
             static const Accessor & instance(); \
             size_t size() const { return field_infos.size(); } \
-            size_t find(const std::string_view & name) const; \
+            size_t find(std::string_view name) const; \
             const String & getName(size_t index) const { return field_infos[index].name; } \
             const char * getTypeName(size_t index) const { return field_infos[index].type; } \
             const char * getDescription(size_t index) const { return field_infos[index].description; } \
@@ -851,7 +851,7 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
     \
     SETTINGS_TRAITS_NAME::Accessor::Accessor() {} \
     \
-    size_t SETTINGS_TRAITS_NAME::Accessor::find(const std::string_view & name) const \
+    size_t SETTINGS_TRAITS_NAME::Accessor::find(std::string_view name) const \
     { \
         auto it = name_to_index_map.find(name); \
         if (it != name_to_index_map.end()) \

src/Core/Field.cpp

@@ -286,7 +286,7 @@ String Field::dump() const
     return applyVisitor(FieldVisitorDump(), *this);
 }
 
-Field Field::restoreFromDump(const std::string_view & dump_)
+Field Field::restoreFromDump(std::string_view dump_)
 {
     auto show_error = [&dump_]
     {

src/Core/Field.h

@@ -346,7 +346,7 @@ public:
     }
 
     /// Create a string inplace.
-    Field(const std::string_view & str) { create(str.data(), str.size()); } /// NOLINT
+    Field(std::string_view str) { create(str.data(), str.size()); } /// NOLINT
     Field(const String & str) { create(std::string_view{str}); } /// NOLINT
     Field(String && str) { create(std::move(str)); } /// NOLINT
     Field(const char * str) { create(std::string_view{str}); } /// NOLINT
@@ -403,7 +403,7 @@ public:
         return *this;
     }
 
-    Field & operator= (const std::string_view & str);
+    Field & operator= (std::string_view str);
     Field & operator= (const String & str) { return *this = std::string_view{str}; }
     Field & operator= (String && str);
     Field & operator= (const char * str) { return *this = std::string_view{str}; }
@@ -631,7 +631,7 @@ public:
     }
 
     String dump() const;
-    static Field restoreFromDump(const std::string_view & dump_);
+    static Field restoreFromDump(std::string_view dump_);
 
 private:
     std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(Types::Which),
@@ -929,7 +929,7 @@ Field::operator=(T && rhs)
     return *this;
 }
 
-inline Field & Field::operator=(const std::string_view & str)
+inline Field & Field::operator=(std::string_view str)
 {
     if (which != Types::String)
     {

src/Core/Settings.h

@@ -96,6 +96,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
     M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \
     M(UInt64, max_replicated_fetches_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated fetches. Zero means unlimited. Only has meaning at server startup.", 0) \
     M(UInt64, max_replicated_sends_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited. Only has meaning at server startup.", 0) \
+    M(UInt64, max_remote_read_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for read. Zero means unlimited. Only has meaning at server startup.", 0) \
+    M(UInt64, max_remote_write_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for write. Zero means unlimited. Only has meaning at server startup.", 0) \
     M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \
     M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \
     \
@@ -286,6 +288,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
     M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \
     M(String, metrics_perf_events_list, "", "Comma separated list of perf metrics that will be measured throughout queries' execution. Empty means all events. See PerfEventInfo in sources for the available events.", 0) \
     M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \
+    M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \
     M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \
     M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \
     \
@@ -391,6 +394,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
     M(Bool, log_query_views, true, "Log query dependent views into system.query_views_log table. This setting have effect only when 'log_queries' is true.", 0) \
     M(String, log_comment, "", "Log comment into system.query_log table and server log. It can be set to arbitrary string no longer than max_query_size.", 0) \
     M(LogsLevel, send_logs_level, LogsLevel::fatal, "Send server text logs with specified minimum level to client. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \
+    M(String, send_logs_source_regexp, "", "Send server text logs with specified regexp to match log source name. Empty means all sources.", 0) \
     M(Bool, enable_optimize_predicate_expression, true, "If it is set to true, optimize predicates to subqueries.", 0) \
     M(Bool, enable_optimize_predicate_expression_to_final_subquery, true, "Allow push predicate to final subquery.", 0) \
     M(Bool, allow_push_predicate_when_subquery_contains_with, true, "Allows push predicate when subquery contains WITH clause", 0) \

src/Core/SettingsFields.cpp

@@ -350,7 +350,7 @@ void SettingFieldURI::readBinary(ReadBuffer & in)
 }
 
 
-void SettingFieldEnumHelpers::writeBinary(const std::string_view & str, WriteBuffer & out)
+void SettingFieldEnumHelpers::writeBinary(std::string_view str, WriteBuffer & out)
 {
     writeStringBinary(str, out);
 }

src/Core/SettingsFields.h

@@ -146,13 +146,13 @@ struct SettingFieldString
     String value;
     bool changed = false;
 
-    explicit SettingFieldString(const std::string_view & str = {}) : value(str) {}
+    explicit SettingFieldString(std::string_view str = {}) : value(str) {}
     explicit SettingFieldString(const String & str) : SettingFieldString(std::string_view{str}) {}
     explicit SettingFieldString(String && str) : value(std::move(str)) {}
     explicit SettingFieldString(const char * str) : SettingFieldString(std::string_view{str}) {}
     explicit SettingFieldString(const Field & f) : SettingFieldString(f.safeGet<const String &>()) {}
 
-    SettingFieldString & operator =(const std::string_view & str) { value = str; changed = true; return *this; }
+    SettingFieldString & operator =(std::string_view str) { value = str; changed = true; return *this; }
     SettingFieldString & operator =(const String & str) { *this = std::string_view{str}; return *this; }
     SettingFieldString & operator =(String && str) { value = std::move(str); changed = true; return *this; }
     SettingFieldString & operator =(const char * str) { *this = std::string_view{str}; return *this; }
@@ -256,7 +256,7 @@ struct SettingFieldEnum
 
 struct SettingFieldEnumHelpers
 {
-    static void writeBinary(const std::string_view & str, WriteBuffer & out);
+    static void writeBinary(std::string_view str, WriteBuffer & out);
     static String readBinary(ReadBuffer & in);
 };
 
@@ -286,7 +286,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
     { \
         using EnumType = ENUM_TYPE; \
         static const String & toString(EnumType value); \
-        static EnumType fromString(const std::string_view & str); \
+        static EnumType fromString(std::string_view str); \
     }; \
     \
     using SettingField##NEW_NAME = SettingFieldEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;
@@ -310,7 +310,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
             ERROR_CODE_FOR_UNEXPECTED_NAME); \
     } \
     \
-    typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(const std::string_view & str) \
+    typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(std::string_view str) \
     { \
         static const std::unordered_map<std::string_view, EnumType> map = [] { \
             std::unordered_map<std::string_view, EnumType> res; \
@@ -430,7 +430,7 @@ void SettingFieldMultiEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
         using EnumType = ENUM_TYPE; \
         static size_t getEnumSize(); \
         static const String & toString(EnumType value); \
-        static EnumType fromString(const std::string_view & str); \
+        static EnumType fromString(std::string_view str); \
     }; \
     \
     using SettingField##NEW_NAME = SettingFieldMultiEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;

src/DataTypes/NestedUtils.cpp

@@ -54,7 +54,7 @@ std::pair<std::string, std::string> splitName(const std::string & name, bool rev
     return {name.substr(0, idx), name.substr(idx + 1)};
 }
 
-std::pair<std::string_view, std::string_view> splitName(const std::string_view & name, bool reverse)
+std::pair<std::string_view, std::string_view> splitName(std::string_view name, bool reverse)
 {
     auto idx = (reverse ? name.find_last_of('.') : name.find_first_of('.'));
     if (idx == std::string::npos || idx == 0 || idx + 1 == name.size())

src/DataTypes/NestedUtils.h

@@ -13,7 +13,7 @@ namespace Nested
 
     /// Splits name of compound identifier by first/last dot (depending on 'reverse' parameter).
     std::pair<std::string, std::string> splitName(const std::string & name, bool reverse = false);
-    std::pair<std::string_view, std::string_view> splitName(const std::string_view & name, bool reverse = false);
+    std::pair<std::string_view, std::string_view> splitName(std::string_view name, bool reverse = false);
 
     /// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot.
     std::string extractTableName(const std::string & nested_name);

src/Disks/DiskDecorator.cpp

@@ -161,6 +161,11 @@ void DiskDecorator::removeSharedFile(const String & path, bool keep_s3)
     delegate->removeSharedFile(path, keep_s3);
 }
 
+void DiskDecorator::removeSharedFileIfExists(const String & path, bool keep_s3)
+{
+    delegate->removeSharedFileIfExists(path, keep_s3);
+}
+
 void DiskDecorator::removeSharedFiles(const RemoveBatchRequest & files, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only)
 {
     delegate->removeSharedFiles(files, keep_all_batch_data, file_names_remove_metadata_only);
@@ -231,4 +236,9 @@ void DiskDecorator::applyNewSettings(const Poco::Util::AbstractConfiguration & c
     delegate->applyNewSettings(config, context, config_prefix, map);
 }
 
+DiskObjectStoragePtr DiskDecorator::createDiskObjectStorage(const String & name)
+{
+    return delegate->createDiskObjectStorage(name);
+}
+
 }

src/Disks/DiskDecorator.h

@@ -52,11 +52,15 @@ public:
 
     void removeFile(const String & path) override;
     void removeFileIfExists(const String & path) override;
+    void removeSharedFileIfExists(const String & path, bool keep_s3) override;
+
     void removeDirectory(const String & path) override;
     void removeRecursive(const String & path) override;
+
     void removeSharedFile(const String & path, bool keep_s3) override;
     void removeSharedRecursive(const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) override;
     void removeSharedFiles(const RemoveBatchRequest & files, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) override;
+
     void setLastModified(const String & path, const Poco::Timestamp & timestamp) override;
     time_t getLastChanged(const String & path) const override;
     Poco::Timestamp getLastModified(const String & path) const override;
@@ -77,9 +81,14 @@ public:
     void shutdown() override;
     void startup(ContextPtr context) override;
     void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override;
+
+    bool supportsCache() const override { return delegate->supportsCache(); }
     String getCacheBasePath() const override { return delegate->getCacheBasePath(); }
-    PathsWithSize getObjectStoragePaths(const String & path) const override { return delegate->getObjectStoragePaths(path); }
-    void getRemotePathsRecursive(const String & path, std::vector<LocalPathWithRemotePaths> & paths_map) override { return delegate->getRemotePathsRecursive(path, paths_map); }
+
+    StoredObjects getStorageObjects(const String & path) const override { return delegate->getStorageObjects(path); }
+    DiskObjectStoragePtr createDiskObjectStorage(const String &) override;
+
+    void getRemotePathsRecursive(const String & path, std::vector<LocalPathWithObjectStoragePaths> & paths_map) override { return delegate->getRemotePathsRecursive(path, paths_map); }
 
     MetadataStoragePtr getMetadataStorage() override { return delegate->getMetadataStorage(); }
 

src/Disks/DiskRestartProxy.cpp

@@ -318,13 +318,14 @@ String DiskRestartProxy::getCacheBasePath() const
     return DiskDecorator::getCacheBasePath();
 }
 
-PathsWithSize DiskRestartProxy::getObjectStoragePaths(const String & path) const
+StoredObjects DiskRestartProxy::getStorageObjects(const String & path) const
 {
     ReadLock lock (mutex);
-    return DiskDecorator::getObjectStoragePaths(path);
+    return DiskDecorator::getStorageObjects(path);
 }
 
-void DiskRestartProxy::getRemotePathsRecursive(const String & path, std::vector<LocalPathWithRemotePaths> & paths_map)
+void DiskRestartProxy::getRemotePathsRecursive(
+    const String & path, std::vector<LocalPathWithObjectStoragePaths> & paths_map)
 {
     ReadLock lock (mutex);
     return DiskDecorator::getRemotePathsRecursive(path, paths_map);

src/Disks/DiskRestartProxy.h

@@ -65,8 +65,8 @@ public:
     String getUniqueId(const String & path) const override;
     bool checkUniqueId(const String & id) const override;
     String getCacheBasePath() const override;
-    PathsWithSize getObjectStoragePaths(const String & path) const override;
-    void getRemotePathsRecursive(const String & path, std::vector<LocalPathWithRemotePaths> & paths_map) override;
+    StoredObjects getStorageObjects(const String & path) const override;
+    void getRemotePathsRecursive(const String & path, std::vector<LocalPathWithObjectStoragePaths> & paths_map) override;
 
     void restart(ContextPtr context);
 

src/Disks/DiskWebServer.cpp

@@ -170,10 +170,10 @@ std::unique_ptr<ReadBufferFromFileBase> DiskWebServer::readFile(const String & p
     auto remote_path = fs_path.parent_path() / (escapeForFileName(fs_path.stem()) + fs_path.extension().string());
     remote_path = remote_path.string().substr(url.size());
 
-    PathsWithSize blobs_to_read;
-    blobs_to_read.emplace_back(remote_path, iter->second.size);
+    StoredObjects objects;
+    objects.emplace_back(remote_path, iter->second.size);
 
-    auto web_impl = std::make_unique<ReadBufferFromWebServerGather>(url, blobs_to_read, getContext(), read_settings);
+    auto web_impl = std::make_unique<ReadBufferFromWebServerGather>(url, objects, getContext(), read_settings);
 
     if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
     {

src/Disks/DiskWebServer.h

@@ -144,6 +144,11 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk {} is read-only", getName());
     }
 
+    void removeSharedFileIfExists(const String &, bool) override
+    {
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk {} is read-only", getName());
+    }
+
     void removeSharedRecursive(const String &, bool, const NameSet &) override
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk {} is read-only", getName());
@@ -169,9 +174,9 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk {} is read-only", getName());
     }
 
-    PathsWithSize getObjectStoragePaths(const String &) const override { return {}; }
+    StoredObjects getStorageObjects(const String &) const override { return {}; }
 
-    void getRemotePathsRecursive(const String &, std::vector<LocalPathWithRemotePaths> &) override {}
+    void getRemotePathsRecursive(const String &, std::vector<LocalPathWithObjectStoragePaths> &) override {}
 
     /// Create part
 

src/Disks/IDisk.cpp

@@ -7,6 +7,8 @@
 #include <Common/logger_useful.h>
 #include <Common/setThreadName.h>
 #include <Disks/ObjectStorages/MetadataStorageFromDisk.h>
+#include <Disks/ObjectStorages/FakeMetadataStorageFromDisk.h>
+#include <Disks/ObjectStorages/LocalObjectStorage.h>
 #include <Disks/FakeDiskTransaction.h>
 
 namespace DB
@@ -119,6 +121,18 @@ SyncGuardPtr IDisk::getDirectorySyncGuard(const String & /* path */) const
     return nullptr;
 }
 
-MetadataStoragePtr IDisk::getMetadataStorage() { return std::make_shared<MetadataStorageFromDisk>(std::static_pointer_cast<IDisk>(shared_from_this()), ""); }
+MetadataStoragePtr IDisk::getMetadataStorage()
+{
+    if (isRemote())
+    {
+        return std::make_shared<MetadataStorageFromDisk>(std::static_pointer_cast<IDisk>(shared_from_this()), "");
+    }
+    else
+    {
+        auto object_storage = std::make_shared<LocalObjectStorage>();
+        return std::make_shared<FakeMetadataStorageFromDisk>(
+            std::static_pointer_cast<IDisk>(shared_from_this()), object_storage, getPath());
+    }
+}
 
 }

src/Disks/IDisk.h

@@ -55,6 +55,8 @@ using DiskTransactionPtr = std::shared_ptr<IDiskTransaction>;
 struct RemoveRequest;
 using RemoveBatchRequest = std::vector<RemoveRequest>;
 
+class DiskObjectStorage;
+using DiskObjectStoragePtr = std::shared_ptr<DiskObjectStorage>;
 
 /**
  * Provide interface for reservation.
@@ -215,19 +217,22 @@ public:
     /// Second bool param is a flag to remove (true) or keep (false) shared data on S3
     virtual void removeSharedFileIfExists(const String & path, bool /* keep_shared_data */) { removeFileIfExists(path); }
 
-    virtual String getCacheBasePath() const { return ""; }
+    virtual String getCacheBasePath() const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no cache path"); }
 
-    /// Returns a list of paths because for Log family engines there might be
-    /// multiple files in remote fs for single clickhouse file.
-    virtual PathsWithSize getObjectStoragePaths(const String &) const
+    virtual bool supportsCache() const { return false; }
+
+    /// Returns a list of storage objects (contains path, size, ...).
+    /// (A list is returned because for Log family engines there might
+    /// be multiple files in remote fs for single clickhouse file.
+    virtual StoredObjects getStorageObjects(const String &) const
     {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getObjectStoragePaths() not implemented for disk: {}`", getType());
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getStorageObjects() not implemented for disk: {}`", getType());
     }
 
     /// For one local path there might be multiple remote paths in case of Log family engines.
-    using LocalPathWithRemotePaths = std::pair<String, PathsWithSize>;
+    using LocalPathWithObjectStoragePaths = std::pair<String, StoredObjects>;
 
-    virtual void getRemotePathsRecursive(const String &, std::vector<LocalPathWithRemotePaths> &)
+    virtual void getRemotePathsRecursive(const String &, std::vector<LocalPathWithObjectStoragePaths> &)
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getRemotePathsRecursive() not implemented for disk: {}`", getType());
     }
@@ -335,6 +340,13 @@ public:
     /// Return current disk revision.
     virtual UInt64 getRevision() const { return 0; }
 
+    virtual DiskObjectStoragePtr createDiskObjectStorage(const String &)
+    {
+        throw Exception(
+            ErrorCodes::NOT_IMPLEMENTED,
+            "Method createDiskObjectStorage() is not implemented for disk type: {}",
+            getType());
+    }
 
 protected:
     friend class DiskDecorator;

src/Disks/IDiskTransaction.h

@@ -10,7 +10,7 @@ namespace DB
 
 struct RemoveRequest
 {
-    std::string path;
+    std::string path; /// Relative path.
     bool if_exists = false;
 
     explicit RemoveRequest(std::string path_, bool if_exists_ = false)

src/Disks/IO/CachedReadBufferFromRemoteFS.cpp

@@ -567,28 +567,31 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment)
 
             ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceBytes, current_impl_buffer_size);
 
-            if (file_segment->reserve(current_predownload_size))
+            bool continue_predownload = file_segment->reserve(current_predownload_size);
+            if (continue_predownload)
             {
                 LOG_TEST(log, "Left to predownload: {}, buffer size: {}", bytes_to_predownload, current_impl_buffer_size);
 
                 assert(file_segment->getDownloadOffset() == static_cast<size_t>(implementation_buffer->getPosition()));
 
-                Stopwatch watch(CLOCK_MONOTONIC);
-
-                file_segment->write(implementation_buffer->buffer().begin(), current_predownload_size, current_offset);
-
-                watch.stop();
-                auto elapsed = watch.elapsedMicroseconds();
-                current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed);
-                ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed);
-                ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, current_predownload_size);
-
+                bool success = writeCache(implementation_buffer->buffer().begin(), current_predownload_size, current_offset, *file_segment);
+                if (success)
+                {
                     current_offset += current_predownload_size;
 
                     bytes_to_predownload -= current_predownload_size;
                     implementation_buffer->position() += current_predownload_size;
                 }
                 else
+                {
+                    read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE;
+                    file_segment->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION);
+
+                    continue_predownload = false;
+                }
+            }
+
+            if (!continue_predownload)
             {
                 /// We were predownloading:
                 ///                   segment{1}
@@ -691,6 +694,34 @@ bool CachedReadBufferFromRemoteFS::updateImplementationBufferIfNeeded()
     return true;
 }
 
+bool CachedReadBufferFromRemoteFS::writeCache(char * data, size_t size, size_t offset, FileSegment & file_segment)
+{
+    Stopwatch watch(CLOCK_MONOTONIC);
+
+    try
+    {
+        file_segment.write(data, size, offset);
+    }
+    catch (ErrnoException & e)
+    {
+        int code = e.getErrno();
+        if (code == /* No space left on device */28 || code == /* Quota exceeded */122)
+        {
+            LOG_INFO(log, "Insert into cache is skipped due to insufficient disk space. ({})", e.displayText());
+            return false;
+        }
+        throw;
+    }
+
+    watch.stop();
+    auto elapsed = watch.elapsedMicroseconds();
+    current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed);
+    ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed);
+    ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, size);
+
+    return true;
+}
+
 bool CachedReadBufferFromRemoteFS::nextImpl()
 {
     try
@@ -840,23 +871,14 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
         {
             assert(file_offset_of_buffer_end + size - 1 <= file_segment->range().right);
 
-            if (file_segment->reserve(size))
+            bool success = file_segment->reserve(size);
+            if (success)
             {
                 assert(file_segment->getDownloadOffset() == static_cast<size_t>(implementation_buffer->getPosition()));
 
-                Stopwatch watch(CLOCK_MONOTONIC);
-
-                file_segment->write(
-                    needed_to_predownload ? implementation_buffer->position() : implementation_buffer->buffer().begin(),
-                    size,
-                    file_offset_of_buffer_end);
-
-                watch.stop();
-                auto elapsed = watch.elapsedMicroseconds();
-                current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed);
-                ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed);
-                ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, size);
-
+                success = writeCache(implementation_buffer->position(), size, file_offset_of_buffer_end, *file_segment);
+                if (success)
+                {
                     assert(file_segment->getDownloadOffset() <= file_segment->range().right + 1);
                     assert(
                         std::next(current_file_segment_it) == file_segments_holder->file_segments.end()
@@ -864,9 +886,19 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
                 }
                 else
                 {
-                download_current_segment = false;
-                file_segment->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION);
+                    assert(file_segment->state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION);
+                }
+            }
+            else
+            {
                 LOG_DEBUG(log, "No space left in cache, will continue without cache download");
+                file_segment->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION);
+            }
+
+            if (!success)
+            {
+                read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE;
+                download_current_segment = false;
             }
         }
 

src/Disks/IO/CachedReadBufferFromRemoteFS.h

@@ -73,10 +73,13 @@ private:
     SeekableReadBufferPtr getRemoteFSReadBuffer(FileSegmentPtr & file_segment, ReadType read_type_);
 
     size_t getTotalSizeToRead();
+
     bool completeFileSegmentAndGetNext();
 
     void appendFilesystemCacheLog(const FileSegment::Range & file_segment_range, ReadType read_type);
 
+    bool writeCache(char * data, size_t size, size_t offset, FileSegment & file_segment);
+
     Poco::Logger * log;
     IFileCache::Key cache_key;
     String remote_fs_object_path;

src/Disks/IO/ReadBufferFromRemoteFSGather.cpp

@@ -87,9 +87,9 @@ SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementation
     return std::make_unique<ReadBufferFromAzureBlobStorage>(
         blob_container_client,
         path,
+        settings,
         max_single_read_retries,
         max_single_download_retries,
-        settings.remote_fs_buffer_size,
         /* use_external_buffer */true,
         read_until_position);
 }
@@ -115,13 +115,13 @@ SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBufferImpl(c
     auto hdfs_uri = path.substr(0, begin_of_path);
     LOG_TEST(log, "HDFS uri: {}, path: {}", hdfs_path, hdfs_uri);
 
-    return std::make_unique<ReadBufferFromHDFS>(hdfs_uri, hdfs_path, config);
+    return std::make_unique<ReadBufferFromHDFS>(hdfs_uri, hdfs_path, config, settings);
 }
 #endif
 
 
 ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(
-    const PathsWithSize & blobs_to_read_,
+    const StoredObjects & blobs_to_read_,
     const ReadSettings & settings_)
     : ReadBuffer(nullptr, 0)
     , blobs_to_read(blobs_to_read_)
@@ -130,6 +130,9 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(
     , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather"))
     , enable_cache_log(!query_id.empty() && settings.enable_filesystem_cache_log)
 {
+    if (blobs_to_read.empty())
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read zero number of objects");
+
     with_cache = settings.remote_fs_cache
         && settings.enable_filesystem_cache
         && (!IFileCache::isReadOnly() || settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache);
@@ -182,22 +185,22 @@ void ReadBufferFromRemoteFSGather::initialize()
     auto current_buf_offset = file_offset_of_buffer_end;
     for (size_t i = 0; i < blobs_to_read.size(); ++i)
     {
-        const auto & [file_path, size] = blobs_to_read[i];
+        const auto & object = blobs_to_read[i];
 
-        if (size > current_buf_offset)
+        if (object.bytes_size > current_buf_offset)
         {
             /// Do not create a new buffer if we already have what we need.
             if (!current_buf || current_buf_idx != i)
             {
                 current_buf_idx = i;
-                current_buf = createImplementationBuffer(file_path, size);
+                current_buf = createImplementationBuffer(object.absolute_path, object.bytes_size);
             }
 
             current_buf->seek(current_buf_offset, SEEK_SET);
             return;
         }
 
-        current_buf_offset -= size;
+        current_buf_offset -= object.bytes_size;
     }
     current_buf_idx = blobs_to_read.size();
     current_buf = nullptr;
@@ -217,10 +220,14 @@ bool ReadBufferFromRemoteFSGather::nextImpl()
             return true;
     }
     else
+    {
         return false;
+    }
 
     if (!moveToNextBuffer())
+    {
         return false;
+    }
 
     return readImpl();
 }
@@ -234,8 +241,8 @@ bool ReadBufferFromRemoteFSGather::moveToNextBuffer()
 
     ++current_buf_idx;
 
-    const auto & [path, size] = blobs_to_read[current_buf_idx];
-    current_buf = createImplementationBuffer(path, size);
+    const auto & object = blobs_to_read[current_buf_idx];
+    current_buf = createImplementationBuffer(object.absolute_path, object.bytes_size);
 
     return true;
 }

src/Disks/IO/ReadBufferFromRemoteFSGather.h

@@ -27,7 +27,7 @@ friend class ReadIndirectBufferFromRemoteFS;
 
 public:
     ReadBufferFromRemoteFSGather(
-        const PathsWithSize & blobs_to_read_,
+        const StoredObjects & blobs_to_read_,
         const ReadSettings & settings_);
 
     ~ReadBufferFromRemoteFSGather() override;
@@ -53,7 +53,7 @@ public:
 protected:
     virtual SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) = 0;
 
-    PathsWithSize blobs_to_read;
+    StoredObjects blobs_to_read;
 
     ReadSettings settings;
 
@@ -109,7 +109,7 @@ public:
         std::shared_ptr<const Aws::S3::S3Client> client_ptr_,
         const String & bucket_,
         const String & version_id_,
-        const PathsWithSize & blobs_to_read_,
+        const StoredObjects & blobs_to_read_,
         size_t max_single_read_retries_,
         const ReadSettings & settings_)
         : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_)
@@ -138,7 +138,7 @@ class ReadBufferFromAzureBlobStorageGather final : public ReadBufferFromRemoteFS
 public:
     ReadBufferFromAzureBlobStorageGather(
         std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> blob_container_client_,
-        const PathsWithSize & blobs_to_read_,
+        const StoredObjects & blobs_to_read_,
         size_t max_single_read_retries_,
         size_t max_single_download_retries_,
         const ReadSettings & settings_)
@@ -164,7 +164,7 @@ class ReadBufferFromWebServerGather final : public ReadBufferFromRemoteFSGather
 public:
     ReadBufferFromWebServerGather(
             const String & uri_,
-            const PathsWithSize & blobs_to_read_,
+            const StoredObjects & blobs_to_read_,
             ContextPtr context_,
             const ReadSettings & settings_)
         : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_)
@@ -188,7 +188,7 @@ class ReadBufferFromHDFSGather final : public ReadBufferFromRemoteFSGather
 public:
     ReadBufferFromHDFSGather(
             const Poco::Util::AbstractConfiguration & config_,
-            const PathsWithSize & blobs_to_read_,
+            const StoredObjects & blobs_to_read_,
             const ReadSettings & settings_)
         : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_)
         , config(config_)

src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp

@@ -5,6 +5,7 @@
 #include <optional>
 #include <re2/re2.h>
 #include <azure/identity/managed_identity_credential.hpp>
+#include <Poco/Util/AbstractConfiguration.h>
 
 using namespace Azure::Storage::Blobs;
 

src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp

@@ -2,6 +2,7 @@
 
 #if USE_AZURE_BLOB_STORAGE
 
+#include <Common/getRandomASCIIString.h>
 #include <IO/ReadBufferFromAzureBlobStorage.h>
 #include <IO/WriteBufferFromAzureBlobStorage.h>
 #include <IO/SeekAvoidingReadBuffer.h>
@@ -9,6 +10,7 @@
 
 #include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h>
 
+
 namespace DB
 {
 
@@ -21,24 +23,27 @@ namespace ErrorCodes
 
 
 AzureObjectStorage::AzureObjectStorage(
-    FileCachePtr && cache_,
     const String & name_,
     AzureClientPtr && client_,
     SettingsPtr && settings_)
-    : IObjectStorage(std::move(cache_))
-    , name(name_)
+    : name(name_)
     , client(std::move(client_))
     , settings(std::move(settings_))
 {
 }
 
-bool AzureObjectStorage::exists(const std::string & uri) const
+std::string AzureObjectStorage::generateBlobNameForPath(const std::string & /* path */)
+{
+    return getRandomASCIIString();
+}
+
+bool AzureObjectStorage::exists(const StoredObject & object) const
 {
     auto client_ptr = client.get();
 
     /// What a shame, no Exists method...
     Azure::Storage::Blobs::ListBlobsOptions options;
-    options.Prefix = uri;
+    options.Prefix = object.absolute_path;
     options.PageSizeHint = 1;
 
     auto blobs_list_response = client_ptr->ListBlobs(options);
@@ -46,15 +51,15 @@ bool AzureObjectStorage::exists(const std::string & uri) const
 
     for (const auto & blob : blobs_list)
     {
-        if (uri == blob.Name)
+        if (object.absolute_path == blob.Name)
             return true;
     }
 
     return false;
 }
 
-std::unique_ptr<SeekableReadBuffer> AzureObjectStorage::readObject( /// NOLINT
-    const std::string & path,
+std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObject( /// NOLINT
+    const StoredObject & object,
     const ReadSettings & read_settings,
     std::optional<size_t>,
     std::optional<size_t>) const
@@ -62,28 +67,29 @@ std::unique_ptr<SeekableReadBuffer> AzureObjectStorage::readObject( /// NOLINT
     auto settings_ptr = settings.get();
 
     return std::make_unique<ReadBufferFromAzureBlobStorage>(
-        client.get(), path, settings_ptr->max_single_read_retries,
-        settings_ptr->max_single_download_retries, read_settings.remote_fs_buffer_size);
+        client.get(), object.absolute_path, patchSettings(read_settings), settings_ptr->max_single_read_retries,
+        settings_ptr->max_single_download_retries);
 }
 
 std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOLINT
-    const PathsWithSize & paths_to_read,
+    const StoredObjects & objects,
     const ReadSettings & read_settings,
     std::optional<size_t>,
     std::optional<size_t>) const
 {
+    ReadSettings disk_read_settings = patchSettings(read_settings);
     auto settings_ptr = settings.get();
     auto reader_impl = std::make_unique<ReadBufferFromAzureBlobStorageGather>(
         client.get(),
-        paths_to_read,
+        objects,
         settings_ptr->max_single_read_retries,
         settings_ptr->max_single_download_retries,
-        read_settings);
+        disk_read_settings);
 
-    if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
+    if (disk_read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
     {
         auto reader = getThreadPoolReader();
-        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, read_settings, std::move(reader_impl));
+        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, disk_read_settings, std::move(reader_impl));
     }
     else
     {
@@ -94,23 +100,24 @@ std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOL
 
 /// Open the file for write and return WriteBufferFromFileBase object.
 std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NOLINT
-    const std::string & path,
+    const StoredObject & object,
     WriteMode mode,
     std::optional<ObjectAttributes>,
     FinalizeCallback && finalize_callback,
     size_t buf_size,
-    const WriteSettings &)
+    const WriteSettings & write_settings)
 {
     if (mode != WriteMode::Rewrite)
         throw Exception("Azure storage doesn't support append", ErrorCodes::UNSUPPORTED_METHOD);
 
     auto buffer = std::make_unique<WriteBufferFromAzureBlobStorage>(
         client.get(),
-        path,
+        object.absolute_path,
         settings.get()->max_single_part_upload_size,
-        buf_size);
+        buf_size,
+        patchSettings(write_settings));
 
-    return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(finalize_callback), path);
+    return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(finalize_callback), object.absolute_path);
 }
 
 void AzureObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const
@@ -128,36 +135,37 @@ void AzureObjectStorage::listPrefix(const std::string & path, RelativePathsWithS
 }
 
 /// Remove file. Throws exception if file doesn't exists or it's a directory.
-void AzureObjectStorage::removeObject(const std::string & path)
+void AzureObjectStorage::removeObject(const StoredObject & object)
 {
+    const auto & path = object.absolute_path;
     auto client_ptr = client.get();
     auto delete_info = client_ptr->DeleteBlob(path);
     if (!delete_info.Value.Deleted)
         throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file in AzureBlob Storage: {}", path);
 }
 
-void AzureObjectStorage::removeObjects(const PathsWithSize & paths)
+void AzureObjectStorage::removeObjects(const StoredObjects & objects)
 {
     auto client_ptr = client.get();
-    for (const auto & [path, _] : paths)
+    for (const auto & object : objects)
     {
-        auto delete_info = client_ptr->DeleteBlob(path);
+        auto delete_info = client_ptr->DeleteBlob(object.absolute_path);
         if (!delete_info.Value.Deleted)
-            throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file in AzureBlob Storage: {}", path);
+            throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file in AzureBlob Storage: {}", object.absolute_path);
     }
 }
 
-void AzureObjectStorage::removeObjectIfExists(const std::string & path)
+void AzureObjectStorage::removeObjectIfExists(const StoredObject & object)
 {
     auto client_ptr = client.get();
-    auto delete_info = client_ptr->DeleteBlob(path);
+    auto delete_info = client_ptr->DeleteBlob(object.absolute_path);
 }
 
-void AzureObjectStorage::removeObjectsIfExist(const PathsWithSize & paths)
+void AzureObjectStorage::removeObjectsIfExist(const StoredObjects & objects)
 {
     auto client_ptr = client.get();
-    for (const auto & [path, _] : paths)
-        auto delete_info = client_ptr->DeleteBlob(path);
+    for (const auto & object : objects)
+        auto delete_info = client_ptr->DeleteBlob(object.absolute_path);
 }
 
 
@@ -179,13 +187,14 @@ ObjectMetadata AzureObjectStorage::getObjectMetadata(const std::string & path) c
 }
 
 void AzureObjectStorage::copyObject( /// NOLINT
-    const std::string & object_from,
-    const std::string & object_to,
+    const StoredObject & object_from,
+    const StoredObject & object_to,
     std::optional<ObjectAttributes> object_to_attributes)
 {
     auto client_ptr = client.get();
-    auto dest_blob_client = client_ptr->GetBlobClient(object_to);
-    auto source_blob_client = client_ptr->GetBlobClient(object_from);
+    auto dest_blob_client = client_ptr->GetBlobClient(object_to.absolute_path);
+    auto source_blob_client = client_ptr->GetBlobClient(object_from.absolute_path);
+
     Azure::Storage::Blobs::CopyBlobFromUriOptions copy_options;
     if (object_to_attributes.has_value())
     {
@@ -200,7 +209,7 @@ void AzureObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguratio
 {
     auto new_settings = getAzureBlobStorageSettings(config, config_prefix, context);
     settings.set(std::move(new_settings));
-
+    applyRemoteThrottlingSettings(context);
     /// We don't update client
 }
 
@@ -208,7 +217,6 @@ void AzureObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguratio
 std::unique_ptr<IObjectStorage> AzureObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context)
 {
     return std::make_unique<AzureObjectStorage>(
-        nullptr,
         name,
         getAzureBlobContainerClient(config, config_prefix),
         getAzureBlobStorageSettings(config, config_prefix, context)

src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h

@@ -45,28 +45,29 @@ public:
     using SettingsPtr = std::unique_ptr<AzureObjectStorageSettings>;
 
     AzureObjectStorage(
-        FileCachePtr && cache_,
         const String & name_,
         AzureClientPtr && client_,
         SettingsPtr && settings_);
 
-    bool exists(const std::string & uri) const override;
+    std::string getName() const override { return "AzureObjectStorage"; }
 
-    std::unique_ptr<SeekableReadBuffer> readObject( /// NOLINT
-        const std::string & path,
+    bool exists(const StoredObject & object) const override;
+
+    std::unique_ptr<ReadBufferFromFileBase> readObject( /// NOLINT
+        const StoredObject & object,
         const ReadSettings & read_settings = ReadSettings{},
         std::optional<size_t> read_hint = {},
         std::optional<size_t> file_size = {}) const override;
 
     std::unique_ptr<ReadBufferFromFileBase> readObjects( /// NOLINT
-        const PathsWithSize & blobs_to_read,
+        const StoredObjects & objects,
         const ReadSettings & read_settings = ReadSettings{},
         std::optional<size_t> read_hint = {},
         std::optional<size_t> file_size = {}) const override;
 
     /// Open the file for write and return WriteBufferFromFileBase object.
     std::unique_ptr<WriteBufferFromFileBase> writeObject( /// NOLINT
-        const std::string & path,
+        const StoredObject & object,
         WriteMode mode,
         std::optional<ObjectAttributes> attributes = {},
         FinalizeCallback && finalize_callback = {},
@@ -76,19 +77,19 @@ public:
     void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
 
     /// Remove file. Throws exception if file doesn't exists or it's a directory.
-    void removeObject(const std::string & path) override;
+    void removeObject(const StoredObject & object) override;
 
-    void removeObjects(const PathsWithSize & paths) override;
+    void removeObjects(const StoredObjects & objects) override;
 
-    void removeObjectIfExists(const std::string & path) override;
+    void removeObjectIfExists(const StoredObject & object) override;
 
-    void removeObjectsIfExist(const PathsWithSize & paths) override;
+    void removeObjectsIfExist(const StoredObjects & objects) override;
 
     ObjectMetadata getObjectMetadata(const std::string & path) const override;
 
     void copyObject( /// NOLINT
-        const std::string & object_from,
-        const std::string & object_to,
+        const StoredObject & object_from,
+        const StoredObject & object_to,
         std::optional<ObjectAttributes> object_to_attributes = {}) override;
 
     void shutdown() override {}
@@ -108,6 +109,9 @@ public:
         const std::string & config_prefix,
         ContextPtr context) override;
 
+    std::string generateBlobNameForPath(const std::string & path) override;
+
+    bool isRemote() const override { return true; }
 
 private:
     const String name;

src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp

@@ -5,7 +5,7 @@
 #if USE_AZURE_BLOB_STORAGE
 
 #include <Disks/DiskRestartProxy.h>
-#include <Disks/DiskCacheWrapper.h>
+
 #include <Disks/ObjectStorages/DiskObjectStorageCommon.h>
 #include <Disks/ObjectStorages/DiskObjectStorage.h>
 
@@ -73,9 +73,7 @@ void registerDiskAzureBlobStorage(DiskFactory & factory)
     {
         auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context);
 
-        /// FIXME Cache currently unsupported :(
         ObjectStoragePtr azure_object_storage = std::make_unique<AzureObjectStorage>(
-            nullptr,
             name,
             getAzureBlobContainerClient(config, config_prefix),
             getAzureBlobStorageSettings(config, config_prefix, context));
@@ -104,6 +102,8 @@ void registerDiskAzureBlobStorage(DiskFactory & factory)
             checkRemoveAccess(*azure_blob_storage_disk);
         }
 
+        azure_blob_storage_disk->startup(context);
+
 #ifdef NDEBUG
         bool use_cache = true;
 #else
@@ -111,17 +111,14 @@ void registerDiskAzureBlobStorage(DiskFactory & factory)
         /// read buffer.
         bool use_cache = false;
 #endif
-
-        azure_blob_storage_disk->startup(context);
-
         if (config.getBool(config_prefix + ".cache_enabled", use_cache))
         {
             String cache_path = config.getString(config_prefix + ".cache_path", context->getPath() + "disks/" + name + "/cache/");
             azure_blob_storage_disk = wrapWithCache(azure_blob_storage_disk, "azure-blob-storage-cache", cache_path, metadata_path);
         }
-
         return std::make_shared<DiskRestartProxy>(azure_blob_storage_disk);
     };
+
     factory.registerDiskType("azure_blob_storage", creator);
 }
 

src/Disks/ObjectStorages/DiskObjectStorage.cpp

@@ -1,4 +1,5 @@
 #include <Disks/ObjectStorages/DiskObjectStorage.h>
+#include <Disks/ObjectStorages/DiskObjectStorageCommon.h>
 
 #include <IO/ReadBufferFromString.h>
 #include <IO/ReadBufferFromFile.h>
@@ -9,7 +10,6 @@
 #include <Common/quoteString.h>
 #include <Common/logger_useful.h>
 #include <Common/checkStackSize.h>
-#include <Common/getRandomASCIIString.h>
 #include <boost/algorithm/string.hpp>
 #include <Common/filesystemHelpers.h>
 #include <Disks/IO/ThreadPoolRemoteFSReader.h>
@@ -89,43 +89,43 @@ DiskTransactionPtr DiskObjectStorage::createObjectStorageTransaction()
     return std::make_shared<DiskObjectStorageTransaction>(
         *object_storage,
         *metadata_storage,
-        remote_fs_root_path,
         send_metadata ? metadata_helper.get() : nullptr);
 }
 
 DiskObjectStorage::DiskObjectStorage(
     const String & name_,
-    const String & remote_fs_root_path_,
+    const String & object_storage_root_path_,
     const String & log_name,
-    MetadataStoragePtr && metadata_storage_,
-    ObjectStoragePtr && object_storage_,
+    MetadataStoragePtr metadata_storage_,
+    ObjectStoragePtr object_storage_,
     DiskType disk_type_,
     bool send_metadata_,
-    uint64_t thread_pool_size)
-    : IDisk(std::make_unique<AsyncThreadPoolExecutor>(log_name, thread_pool_size))
+    uint64_t thread_pool_size_)
+    : IDisk(std::make_unique<AsyncThreadPoolExecutor>(log_name, thread_pool_size_))
     , name(name_)
-    , remote_fs_root_path(remote_fs_root_path_)
-    , log (&Poco::Logger::get(log_name))
+    , object_storage_root_path(object_storage_root_path_)
+    , log (&Poco::Logger::get("DiskObjectStorage(" + log_name + ")"))
     , disk_type(disk_type_)
     , metadata_storage(std::move(metadata_storage_))
     , object_storage(std::move(object_storage_))
     , send_metadata(send_metadata_)
+    , threadpool_size(thread_pool_size_)
     , metadata_helper(std::make_unique<DiskObjectStorageRemoteMetadataRestoreHelper>(this, ReadSettings{}))
 {}
 
-PathsWithSize DiskObjectStorage::getObjectStoragePaths(const String & local_path) const
+StoredObjects DiskObjectStorage::getStorageObjects(const String & local_path) const
 {
-    return metadata_storage->getObjectStoragePaths(local_path);
+    return metadata_storage->getStorageObjects(local_path);
 }
 
-void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std::vector<LocalPathWithRemotePaths> & paths_map)
+void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std::vector<LocalPathWithObjectStoragePaths> & paths_map)
 {
     /// Protect against concurrent delition of files (for example because of a merge).
     if (metadata_storage->isFile(local_path))
     {
         try
         {
-            paths_map.emplace_back(local_path, getObjectStoragePaths(local_path));
+            paths_map.emplace_back(local_path, getStorageObjects(local_path));
         }
         catch (const Exception & e)
         {
@@ -247,25 +247,20 @@ std::unordered_map<String, String> DiskObjectStorage::getSerializedMetadata(cons
 
 String DiskObjectStorage::getUniqueId(const String & path) const
 {
-    LOG_TRACE(log, "Remote path: {}, Path: {}", remote_fs_root_path, path);
     String id;
-    auto blobs_paths = metadata_storage->getObjectStoragePaths(path);
+    auto blobs_paths = metadata_storage->getStorageObjects(path);
     if (!blobs_paths.empty())
-        id = blobs_paths[0].path;
+        id = blobs_paths[0].absolute_path;
     return id;
 }
 
-bool DiskObjectStorage::checkObjectExists(const String & path) const
-{
-    if (!path.starts_with(remote_fs_root_path))
-        return false;
-
-    return object_storage->exists(path);
-}
-
 bool DiskObjectStorage::checkUniqueId(const String & id) const
 {
-    return checkObjectExists(id);
+    if (!id.starts_with(object_storage_root_path))
+        return false;
+
+    auto object = StoredObject::create(*object_storage, id, {}, true);
+    return object_storage->exists(object);
 }
 
 void DiskObjectStorage::createHardLink(const String & src_path, const String & dst_path, bool should_send_metadata)
@@ -394,7 +389,8 @@ ReservationPtr DiskObjectStorage::reserve(UInt64 bytes)
     if (!tryReserve(bytes))
         return {};
 
-    return std::make_unique<DiskObjectStorageReservation>(std::static_pointer_cast<DiskObjectStorage>(shared_from_this()), bytes);
+    return std::make_unique<DiskObjectStorageReservation>(
+        std::static_pointer_cast<DiskObjectStorage>(shared_from_this()), bytes);
 }
 
 void DiskObjectStorage::removeSharedFileIfExists(const String & path, bool delete_metadata_only)
@@ -404,7 +400,8 @@ void DiskObjectStorage::removeSharedFileIfExists(const String & path, bool delet
     transaction->commit();
 }
 
-void DiskObjectStorage::removeSharedRecursive(const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only)
+void DiskObjectStorage::removeSharedRecursive(
+    const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only)
 {
     auto transaction = createObjectStorageTransaction();
     transaction->removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only);
@@ -437,6 +434,24 @@ std::optional<UInt64> DiskObjectStorage::tryReserve(UInt64 bytes)
     return {};
 }
 
+bool DiskObjectStorage::supportsCache() const
+{
+    return object_storage->supportsCache();
+}
+
+DiskObjectStoragePtr DiskObjectStorage::createDiskObjectStorage(const String & name_)
+{
+    return std::make_shared<DiskObjectStorage>(
+        name_,
+        object_storage_root_path,
+        name,
+        metadata_storage,
+        object_storage,
+        disk_type,
+        send_metadata,
+        threadpool_size);
+}
+
 std::unique_ptr<ReadBufferFromFileBase> DiskObjectStorage::readFile(
     const String & path,
     const ReadSettings & settings,
@@ -444,7 +459,7 @@ std::unique_ptr<ReadBufferFromFileBase> DiskObjectStorage::readFile(
     std::optional<size_t> file_size) const
 {
     return object_storage->readObjects(
-        metadata_storage->getObjectStoragePaths(path),
+        metadata_storage->getStorageObjects(path),
         settings,
         read_hint,
         file_size);
@@ -456,13 +471,16 @@ std::unique_ptr<WriteBufferFromFileBase> DiskObjectStorage::writeFile(
     WriteMode mode,
     const WriteSettings & settings)
 {
+    LOG_TEST(log, "Write file: {}", path);
+
     auto transaction = createObjectStorageTransaction();
     auto result = transaction->writeFile(path, buf_size, mode, settings);
 
     return result;
 }
 
-void DiskObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context_, const String &, const DisksMap &)
+void DiskObjectStorage::applyNewSettings(
+    const Poco::Util::AbstractConfiguration & config, ContextPtr context_, const String &, const DisksMap &)
 {
     const auto config_prefix = "storage_configuration.disks." + name;
     object_storage->applyNewSettings(config, config_prefix, context_);
@@ -471,13 +489,15 @@ void DiskObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration
         exec->setMaxThreads(config.getInt(config_prefix + ".thread_pool_size", 16));
 }
 
-void DiskObjectStorage::restoreMetadataIfNeeded(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context)
+void DiskObjectStorage::restoreMetadataIfNeeded(
+    const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context)
 {
     if (send_metadata)
     {
         metadata_helper->restore(config, config_prefix, context);
 
-        if (metadata_helper->readSchemaVersion(object_storage.get(), remote_fs_root_path) < DiskObjectStorageRemoteMetadataRestoreHelper::RESTORABLE_SCHEMA_VERSION)
+        auto current_schema_version = metadata_helper->readSchemaVersion(object_storage.get(), object_storage_root_path);
+        if (current_schema_version < DiskObjectStorageRemoteMetadataRestoreHelper::RESTORABLE_SCHEMA_VERSION)
             metadata_helper->migrateToRestorableSchema();
 
         metadata_helper->findLastRevision();

src/Disks/ObjectStorages/DiskObjectStorage.h

@@ -2,6 +2,7 @@
 
 #include <Disks/IDisk.h>
 #include <Disks/ObjectStorages/IObjectStorage.h>
+#include <Common/FileCache_fwd.h>
 #include <Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.h>
 #include <Disks/ObjectStorages/IMetadataStorage.h>
 #include <Disks/ObjectStorages/DiskObjectStorageTransaction.h>
@@ -29,13 +30,13 @@ friend class DiskObjectStorageRemoteMetadataRestoreHelper;
 public:
     DiskObjectStorage(
         const String & name_,
-        const String & remote_fs_root_path_,
+        const String & object_storage_root_path_,
         const String & log_name,
-        MetadataStoragePtr && metadata_storage_,
-        ObjectStoragePtr && object_storage_,
+        MetadataStoragePtr metadata_storage_,
+        ObjectStoragePtr object_storage_,
         DiskType disk_type_,
         bool send_metadata_,
-        uint64_t thread_pool_size);
+        uint64_t thread_pool_size_);
 
     /// Create fake transaction
     DiskTransactionPtr createTransaction() override;
@@ -50,9 +51,9 @@ public:
 
     const String & getPath() const override { return metadata_storage->getPath(); }
 
-    PathsWithSize getObjectStoragePaths(const String & local_path) const override;
+    StoredObjects getStorageObjects(const String & local_path) const override;
 
-    void getRemotePathsRecursive(const String & local_path, std::vector<LocalPathWithRemotePaths> & paths_map) override;
+    void getRemotePathsRecursive(const String & local_path, std::vector<LocalPathWithObjectStoragePaths> & paths_map) override;
 
     std::string getCacheBasePath() const override
     {
@@ -104,7 +105,6 @@ public:
 
     String getUniqueId(const String & path) const override;
 
-    bool checkObjectExists(const String & path) const;
     bool checkUniqueId(const String & id) const override;
 
     void createHardLink(const String & src_path, const String & dst_path) override;
@@ -163,6 +163,11 @@ public:
     void syncRevision(UInt64 revision) override;
 
     UInt64 getRevision() const override;
+
+    DiskObjectStoragePtr createDiskObjectStorage(const String & name_) override;
+
+    bool supportsCache() const override;
+
 private:
 
     /// Create actual disk object storage transaction for operations
@@ -170,7 +175,7 @@ private:
     DiskTransactionPtr createObjectStorageTransaction();
 
     const String name;
-    const String remote_fs_root_path;
+    const String object_storage_root_path;
     Poco::Logger * log;
 
     const DiskType disk_type;
@@ -184,10 +189,13 @@ private:
     std::optional<UInt64> tryReserve(UInt64 bytes);
 
     const bool send_metadata;
+    size_t threadpool_size;
 
     std::unique_ptr<DiskObjectStorageRemoteMetadataRestoreHelper> metadata_helper;
 };
 
+using DiskObjectStoragePtr = std::shared_ptr<DiskObjectStorage>;
+
 class DiskObjectStorageReservation final : public IReservation
 {
 public:
@@ -210,7 +218,7 @@ public:
     ~DiskObjectStorageReservation() override;
 
 private:
-    std::shared_ptr<DiskObjectStorage> disk;
+    DiskObjectStoragePtr disk;
     UInt64 size;
     UInt64 unreserved_space;
     CurrentMetrics::Increment metric_increment;

src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp

@@ -4,6 +4,8 @@
 #include <IO/WriteBufferFromString.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <IO/WriteBufferFromFileBase.h>
+#include <Common/logger_useful.h>
 
 namespace DB
 {
@@ -36,23 +38,24 @@ void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf)
 
     for (size_t i = 0; i < storage_objects_count; ++i)
     {
-        String remote_fs_object_path;
-        size_t remote_fs_object_size;
-        readIntText(remote_fs_object_size, buf);
+        String object_relative_path;
+        size_t object_size;
+        readIntText(object_size, buf);
         assertChar('\t', buf);
-        readEscapedString(remote_fs_object_path, buf);
+        readEscapedString(object_relative_path, buf);
         if (version == VERSION_ABSOLUTE_PATHS)
         {
-            if (!remote_fs_object_path.starts_with(remote_fs_root_path))
+            if (!object_relative_path.starts_with(object_storage_root_path))
                 throw Exception(ErrorCodes::UNKNOWN_FORMAT,
                     "Path in metadata does not correspond to root path. Path: {}, root path: {}, disk path: {}",
-                    remote_fs_object_path, remote_fs_root_path, common_metadata_path);
+                    object_relative_path, object_storage_root_path, common_metadata_path);
 
-            remote_fs_object_path = remote_fs_object_path.substr(remote_fs_root_path.size());
+            object_relative_path = object_relative_path.substr(object_storage_root_path.size());
         }
         assertChar('\n', buf);
-        storage_objects[i].path = remote_fs_object_path;
-        storage_objects[i].bytes_size = remote_fs_object_size;
+
+        storage_objects[i].relative_path = object_relative_path;
+        storage_objects[i].bytes_size = object_size;
     }
 
     readIntText(ref_count, buf);
@@ -81,11 +84,11 @@ void DiskObjectStorageMetadata::serialize(WriteBuffer & buf, bool sync) const
     writeIntText(total_size, buf);
     writeChar('\n', buf);
 
-    for (const auto & [remote_fs_object_path, remote_fs_object_size] : storage_objects)
+    for (const auto & [object_relative_path, object_size] : storage_objects)
     {
-        writeIntText(remote_fs_object_size, buf);
+        writeIntText(object_size, buf);
         writeChar('\t', buf);
-        writeEscapedString(remote_fs_object_path, buf);
+        writeEscapedString(object_relative_path, buf);
         writeChar('\n', buf);
     }
 
@@ -110,17 +113,17 @@ std::string DiskObjectStorageMetadata::serializeToString() const
 /// Load metadata by path or create empty if `create` flag is set.
 DiskObjectStorageMetadata::DiskObjectStorageMetadata(
         const std::string & common_metadata_path_,
-        const String & remote_fs_root_path_,
+        const String & object_storage_root_path_,
         const String & metadata_file_path_)
     : common_metadata_path(common_metadata_path_)
-    , remote_fs_root_path(remote_fs_root_path_)
+    , object_storage_root_path(object_storage_root_path_)
     , metadata_file_path(metadata_file_path_)
 {
 }
 
 void DiskObjectStorageMetadata::addObject(const String & path, size_t size)
 {
-    if (!remote_fs_root_path.empty() && path.starts_with(remote_fs_root_path))
+    if (!object_storage_root_path.empty() && path.starts_with(object_storage_root_path))
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected relative path");
 
     total_size += size;

src/Disks/ObjectStorages/DiskObjectStorageMetadata.h

@@ -22,8 +22,7 @@ private:
     /// Relative paths of blobs.
     RelativePathsWithSize storage_objects;
 
-    /// URI
-    const std::string & remote_fs_root_path;
+    const std::string object_storage_root_path;
 
     /// Relative path to metadata file on local FS.
     const std::string metadata_file_path;
@@ -44,7 +43,7 @@ public:
 
     DiskObjectStorageMetadata(
         const std::string & common_metadata_path_,
-        const std::string & remote_fs_root_path_,
+        const std::string & object_storage_root_path_,
         const std::string & metadata_file_path_);
 
     void addObject(const std::string & path, size_t size);
@@ -57,7 +56,7 @@ public:
 
     std::string getBlobsCommonPrefix() const
     {
-        return remote_fs_root_path;
+        return object_storage_root_path;
     }
 
     RelativePathsWithSize getBlobsRelativePaths() const

src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp

@@ -5,6 +5,7 @@
 #include <IO/ReadHelpers.h>
 #include <IO/ReadBufferFromFile.h>
 #include <IO/ReadBufferFromString.h>
+#include <IO/WriteBufferFromFile.h>
 #include <Common/checkStackSize.h>
 #include <Common/logger_useful.h>
 
@@ -23,10 +24,12 @@ static String revisionToString(UInt64 revision)
     return std::bitset<64>(revision).to_string();
 }
 
-void DiskObjectStorageRemoteMetadataRestoreHelper::createFileOperationObject(const String & operation_name, UInt64 revision, const ObjectAttributes & metadata) const
+void DiskObjectStorageRemoteMetadataRestoreHelper::createFileOperationObject(
+    const String & operation_name, UInt64 revision, const ObjectAttributes & metadata) const
 {
-    const String path = disk->remote_fs_root_path + "operations/r" + revisionToString(revision) + operation_log_suffix + "-" + operation_name;
-    auto buf = disk->object_storage->writeObject(path, WriteMode::Rewrite, metadata);
+    const String relative_path = "operations/r" + revisionToString(revision) + operation_log_suffix + "-" + operation_name;
+    StoredObject object(fs::path(disk->object_storage_root_path) / relative_path);
+    auto buf = disk->object_storage->writeObject(object, WriteMode::Rewrite, metadata);
     buf->write('0');
     buf->finalize();
 }
@@ -42,9 +45,12 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::findLastRevision()
 
         LOG_TRACE(disk->log, "Check object exists with revision prefix {}", revision_prefix);
 
+        const auto & object_storage = disk->object_storage;
+        StoredObject revision_object{disk->object_storage_root_path + "r" + revision_prefix};
+        StoredObject revision_operation_object{disk->object_storage_root_path + "operations/r" + revision_prefix};
+
         /// Check file or operation with such revision prefix exists.
-        if (disk->object_storage->exists(disk->remote_fs_root_path + "r" + revision_prefix)
-            || disk->object_storage->exists(disk->remote_fs_root_path + "operations/r" + revision_prefix))
+        if (object_storage->exists(revision_object) || object_storage->exists(revision_operation_object))
             revision += "1";
         else
             revision += "0";
@@ -55,12 +61,12 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::findLastRevision()
 
 int DiskObjectStorageRemoteMetadataRestoreHelper::readSchemaVersion(IObjectStorage * object_storage, const String & source_path)
 {
-    const std::string path = source_path + SCHEMA_VERSION_OBJECT;
+    StoredObject object(fs::path(source_path) / SCHEMA_VERSION_OBJECT);
     int version = 0;
-    if (!object_storage->exists(path))
+    if (!object_storage->exists(object))
         return version;
 
-    auto buf = object_storage->readObject(path);
+    auto buf = object_storage->readObject(object);
     readIntText(version, *buf);
 
     return version;
@@ -68,9 +74,9 @@ int DiskObjectStorageRemoteMetadataRestoreHelper::readSchemaVersion(IObjectStora
 
 void DiskObjectStorageRemoteMetadataRestoreHelper::saveSchemaVersion(const int & version) const
 {
-    auto path = disk->remote_fs_root_path + SCHEMA_VERSION_OBJECT;
+    StoredObject object{fs::path(disk->object_storage_root_path) / SCHEMA_VERSION_OBJECT};
 
-    auto buf = disk->object_storage->writeObject(path, WriteMode::Rewrite);
+    auto buf = disk->object_storage->writeObject(object, WriteMode::Rewrite);
     writeIntText(version, *buf);
     buf->finalize();
 
@@ -78,20 +84,21 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::saveSchemaVersion(const int &
 
 void DiskObjectStorageRemoteMetadataRestoreHelper::updateObjectMetadata(const String & key, const ObjectAttributes & metadata) const
 {
-    disk->object_storage->copyObject(key, key, metadata);
+    StoredObject object{key};
+    disk->object_storage->copyObject(object, object, metadata);
 }
 
 void DiskObjectStorageRemoteMetadataRestoreHelper::migrateFileToRestorableSchema(const String & path) const
 {
     LOG_TRACE(disk->log, "Migrate file {} to restorable schema", disk->metadata_storage->getPath() + path);
 
-    auto objects = disk->metadata_storage->getObjectStoragePaths(path);
-    for (const auto & [object_path, _] : objects)
+    auto objects = disk->metadata_storage->getStorageObjects(path);
+    for (const auto & object : objects)
     {
         ObjectAttributes metadata {
             {"path", path}
         };
-        updateObjectMetadata(object_path, metadata);
+        updateObjectMetadata(object.absolute_path, metadata);
     }
 }
 void DiskObjectStorageRemoteMetadataRestoreHelper::migrateToRestorableSchemaRecursive(const String & path, Futures & results)
@@ -180,7 +187,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restore(const Poco::Util::Abs
     try
     {
         RestoreInformation information;
-        information.source_path = disk->remote_fs_root_path;
+        information.source_path = disk->object_storage_root_path;
         information.source_namespace = disk->object_storage->getObjectsNamespace();
 
         readRestoreInformation(information);
@@ -194,12 +201,14 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restore(const Poco::Util::Abs
         {
             /// In this case we need to additionally cleanup S3 from objects with later revision.
             /// Will be simply just restore to different path.
-            if (information.source_path == disk->remote_fs_root_path && information.revision != LATEST_REVISION)
+            if (information.source_path == disk->object_storage_root_path && information.revision != LATEST_REVISION)
                 throw Exception("Restoring to the same bucket and path is allowed if revision is latest (0)", ErrorCodes::BAD_ARGUMENTS);
 
             /// This case complicates S3 cleanup in case of unsuccessful restore.
-            if (information.source_path != disk->remote_fs_root_path && disk->remote_fs_root_path.starts_with(information.source_path))
-                throw Exception("Restoring to the same bucket is allowed only if source path is not a sub-path of configured path in S3 disk", ErrorCodes::BAD_ARGUMENTS);
+            if (information.source_path != disk->object_storage_root_path && disk->object_storage_root_path.starts_with(information.source_path))
+                throw Exception(
+                    ErrorCodes::BAD_ARGUMENTS,
+                    "Restoring to the same bucket is allowed only if source path is not a sub-path of configured path in S3 disk");
         }
         else
         {
@@ -215,7 +224,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restore(const Poco::Util::Abs
 
         LOG_INFO(disk->log, "Removing old metadata...");
 
-        bool cleanup_s3 = information.source_path != disk->remote_fs_root_path;
+        bool cleanup_s3 = information.source_path != disk->object_storage_root_path;
         for (const auto & root : data_roots)
             if (disk->exists(root))
                 disk->removeSharedRecursive(root + '/', !cleanup_s3, {});
@@ -347,24 +356,24 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage *
     LOG_INFO(disk->log, "Starting restore files for disk {}", disk->name);
 
     std::vector<std::future<void>> results;
-    auto restore_files = [this, &source_object_storage, &restore_information, &results](const PathsWithSize & keys)
+    auto restore_files = [this, &source_object_storage, &restore_information, &results](const RelativePathsWithSize & objects)
     {
         std::vector<String> keys_names;
-        for (const auto & [key, size] : keys)
+        for (const auto & object : objects)
         {
 
-            LOG_INFO(disk->log, "Calling restore for key for disk {}", key);
+            LOG_INFO(disk->log, "Calling restore for key for disk {}", object.relative_path);
 
             /// Skip file operations objects. They will be processed separately.
-            if (key.find("/operations/") != String::npos)
+            if (object.relative_path.find("/operations/") != String::npos)
                 continue;
 
-            const auto [revision, _] = extractRevisionAndOperationFromKey(key);
+            const auto [revision, _] = extractRevisionAndOperationFromKey(object.relative_path);
             /// Filter early if it's possible to get revision from key.
             if (revision > restore_information.revision)
                 continue;
 
-            keys_names.push_back(key);
+            keys_names.push_back(object.relative_path);
         }
 
         if (!keys_names.empty())
@@ -394,7 +403,8 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage *
 
 }
 
-void DiskObjectStorageRemoteMetadataRestoreHelper::processRestoreFiles(IObjectStorage * source_object_storage, const String & source_path, const std::vector<String> & keys) const
+void DiskObjectStorageRemoteMetadataRestoreHelper::processRestoreFiles(
+    IObjectStorage * source_object_storage, const String & source_path, const std::vector<String> & keys) const
 {
     for (const auto & key : keys)
     {
@@ -421,9 +431,12 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::processRestoreFiles(IObjectSt
         disk->createDirectories(directoryPath(path));
         auto relative_key = shrinkKey(source_path, key);
 
+        StoredObject object_from{key};
+        StoredObject object_to{fs::path(disk->object_storage_root_path) / relative_key};
+
         /// Copy object if we restore to different bucket / path.
-        if (source_object_storage->getObjectsNamespace() != disk->object_storage->getObjectsNamespace() || disk->remote_fs_root_path != source_path)
-            source_object_storage->copyObjectToAnotherObjectStorage(key, disk->remote_fs_root_path + relative_key, *disk->object_storage);
+        if (source_object_storage->getObjectsNamespace() != disk->object_storage->getObjectsNamespace() || disk->object_storage_root_path != source_path)
+            source_object_storage->copyObjectToAnotherObjectStorage(object_from, object_to, *disk->object_storage);
 
         auto tx = disk->metadata_storage->createTransaction();
         tx->addBlobToMetadata(path, relative_key, meta.size_bytes);
@@ -454,20 +467,21 @@ static String pathToDetached(const String & source_path)
 void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObjectStorage * source_object_storage, const RestoreInformation & restore_information)
 {
     /// Enable recording file operations if we restore to different bucket / path.
-    bool send_metadata = source_object_storage->getObjectsNamespace() != disk->object_storage->getObjectsNamespace() || disk->remote_fs_root_path != restore_information.source_path;
+    bool send_metadata = source_object_storage->getObjectsNamespace() != disk->object_storage->getObjectsNamespace()
+        || disk->object_storage_root_path != restore_information.source_path;
 
     std::set<String> renames;
-    auto restore_file_operations = [this, &source_object_storage, &restore_information, &renames, &send_metadata](const PathsWithSize & keys)
+    auto restore_file_operations = [this, &source_object_storage, &restore_information, &renames, &send_metadata](const RelativePathsWithSize & objects)
     {
         const String rename = "rename";
         const String hardlink = "hardlink";
 
-        for (const auto & [key, _]: keys)
+        for (const auto & object : objects)
         {
-            const auto [revision, operation] = extractRevisionAndOperationFromKey(key);
+            const auto [revision, operation] = extractRevisionAndOperationFromKey(object.relative_path);
             if (revision == UNKNOWN_REVISION)
             {
-                LOG_WARNING(disk->log, "Skip key {} with unknown revision", key);
+                LOG_WARNING(disk->log, "Skip key {} with unknown revision", object.relative_path);
                 continue;
             }
 
@@ -480,7 +494,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject
             if (send_metadata)
                 revision_counter = revision - 1;
 
-            auto object_attributes = *(source_object_storage->getObjectMetadata(key).attributes);
+            auto object_attributes = *(source_object_storage->getObjectMetadata(object.relative_path).attributes);
             if (operation == rename)
             {
                 auto from_path = object_attributes["from_path"];

src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp

@@ -22,12 +22,10 @@ namespace ErrorCodes
 DiskObjectStorageTransaction::DiskObjectStorageTransaction(
     IObjectStorage & object_storage_,
     IMetadataStorage & metadata_storage_,
-    const std::string & remote_fs_root_path_,
     DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_)
     : object_storage(object_storage_)
     , metadata_storage(metadata_storage_)
     , metadata_transaction(metadata_storage.createTransaction())
-    , remote_fs_root_path(remote_fs_root_path_)
     , metadata_helper(metadata_helper_)
 {}
 
@@ -59,15 +57,17 @@ struct PureMetadataObjectStorageOperation final : public IDiskObjectStorageOpera
     void finalize() override
     {
     }
+
+    std::string getInfoForLog() const override { return fmt::format("PureMetadataObjectStorageOperation"); }
 };
 
 struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation
 {
     std::string path;
     bool delete_metadata_only;
-    bool remove_from_cache{false};
-    PathsWithSize paths_to_remove;
+    StoredObjects objects_to_remove;
     bool if_exists;
+    bool remove_from_cache = false;
 
     RemoveObjectStorageOperation(
         IObjectStorage & object_storage_,
@@ -81,6 +81,11 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation
         , if_exists(if_exists_)
     {}
 
+    std::string getInfoForLog() const override
+    {
+        return fmt::format("RemoveObjectStorageOperation (path: {}, if exists: {})", path, if_exists);
+    }
+
     void execute(MetadataTransactionPtr tx) override
     {
         if (!metadata_storage.exists(path))
@@ -97,13 +102,13 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation
         try
         {
             uint32_t hardlink_count = metadata_storage.getHardlinkCount(path);
-            auto objects = metadata_storage.getObjectStoragePaths(path);
+            auto objects = metadata_storage.getStorageObjects(path);
 
             tx->unlinkMetadata(path);
 
             if (hardlink_count == 0)
             {
-                paths_to_remove = objects;
+                objects_to_remove = objects;
                 remove_from_cache = true;
             }
         }
@@ -129,25 +134,24 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation
 
     void finalize() override
     {
-        if (!delete_metadata_only && !paths_to_remove.empty())
-            object_storage.removeObjects(paths_to_remove);
+        if (!delete_metadata_only && !objects_to_remove.empty())
+            object_storage.removeObjects(objects_to_remove);
 
         if (remove_from_cache)
         {
-            for (const auto & path_to_remove : paths_to_remove)
-                object_storage.removeFromCache(path_to_remove.path);
+            for (const auto & object : objects_to_remove)
+                object_storage.removeCacheIfExists(object.getPathKeyForCache());
         }
-
     }
 };
 
 struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOperation
 {
     std::string path;
-    std::unordered_map<std::string, PathsWithSize> paths_to_remove;
+    std::unordered_map<std::string, StoredObjects> objects_to_remove;
     bool keep_all_batch_data;
     NameSet file_names_remove_metadata_only;
-    PathsWithSize path_to_remove_from_cache;
+    StoredObjects objects_to_remove_from_cache;
 
     RemoveRecursiveObjectStorageOperation(
         IObjectStorage & object_storage_,
@@ -161,6 +165,11 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp
         , file_names_remove_metadata_only(file_names_remove_metadata_only_)
     {}
 
+    std::string getInfoForLog() const override
+    {
+        return fmt::format("RemoveRecursiveObjectStorageOperation (path: {})", path);
+    }
+
     void removeMetadataRecursive(MetadataTransactionPtr tx, const std::string & path_to_remove)
     {
         checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks.
@@ -170,14 +179,14 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp
             try
             {
                 uint32_t hardlink_count = metadata_storage.getHardlinkCount(path_to_remove);
-                auto objects_paths = metadata_storage.getObjectStoragePaths(path_to_remove);
+                auto objects_paths = metadata_storage.getStorageObjects(path_to_remove);
 
                 tx->unlinkMetadata(path_to_remove);
 
                 if (hardlink_count == 0)
                 {
-                    paths_to_remove[path_to_remove] = objects_paths;
-                    path_to_remove_from_cache.insert(path_to_remove_from_cache.end(), objects_paths.begin(), objects_paths.end());
+                    objects_to_remove[path_to_remove] = objects_paths;
+                    objects_to_remove_from_cache.insert(objects_to_remove_from_cache.end(), objects_paths.begin(), objects_paths.end());
                 }
 
             }
@@ -218,8 +227,8 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp
     {
         if (!keep_all_batch_data)
         {
-            PathsWithSize remove_from_remote;
-            for (auto && [local_path, remote_paths] : paths_to_remove)
+            StoredObjects remove_from_remote;
+            for (auto && [local_path, remote_paths] : objects_to_remove)
             {
                 if (!file_names_remove_metadata_only.contains(fs::path(local_path).filename()))
                 {
@@ -229,8 +238,8 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp
             object_storage.removeObjects(remove_from_remote);
         }
 
-        for (const auto & [path_to_remove, _] : path_to_remove_from_cache)
-            object_storage.removeFromCache(path_to_remove);
+        for (const auto & object : objects_to_remove_from_cache)
+            object_storage.removeCacheIfExists(object.getPathKeyForCache());
     }
 };
 
@@ -239,7 +248,7 @@ struct ReplaceFileObjectStorageOperation final : public IDiskObjectStorageOperat
 {
     std::string path_from;
     std::string path_to;
-    PathsWithSize blobs_to_remove;
+    StoredObjects objects_to_remove;
 
     ReplaceFileObjectStorageOperation(
         IObjectStorage & object_storage_,
@@ -251,11 +260,16 @@ struct ReplaceFileObjectStorageOperation final : public IDiskObjectStorageOperat
         , path_to(path_to_)
     {}
 
+    std::string getInfoForLog() const override
+    {
+        return fmt::format("ReplaceFileObjectStorageOperation (path_from: {}, path_to: {})", path_from, path_to);
+    }
+
     void execute(MetadataTransactionPtr tx) override
     {
         if (metadata_storage.exists(path_to))
         {
-            blobs_to_remove = metadata_storage.getObjectStoragePaths(path_to);
+            objects_to_remove = metadata_storage.getStorageObjects(path_to);
             tx->replaceFile(path_from, path_to);
         }
         else
@@ -269,28 +283,29 @@ struct ReplaceFileObjectStorageOperation final : public IDiskObjectStorageOperat
 
     void finalize() override
     {
-        if (!blobs_to_remove.empty())
-            object_storage.removeObjects(blobs_to_remove);
+        if (!objects_to_remove.empty())
+            object_storage.removeObjects(objects_to_remove);
     }
 };
 
 struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperation
 {
-    std::string path;
-    std::string blob_path;
-    size_t size;
+    StoredObject object;
     std::function<void(MetadataTransactionPtr)> on_execute;
 
     WriteFileObjectStorageOperation(
         IObjectStorage & object_storage_,
         IMetadataStorage & metadata_storage_,
-        const std::string & path_,
-        const std::string & blob_path_)
+        const StoredObject & object_)
         : IDiskObjectStorageOperation(object_storage_, metadata_storage_)
-        , path(path_)
-        , blob_path(blob_path_)
+        , object(object_)
     {}
 
+    std::string getInfoForLog() const override
+    {
+        return fmt::format("WriteFileObjectStorageOperation");
+    }
+
     void setOnExecute(std::function<void(MetadataTransactionPtr)> && on_execute_)
     {
         on_execute = on_execute_;
@@ -304,8 +319,8 @@ struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperatio
 
     void undo() override
     {
-        if (object_storage.exists(blob_path))
-            object_storage.removeObject(blob_path);
+        if (object_storage.exists(object))
+            object_storage.removeObject(object);
     }
 
     void finalize() override
@@ -316,47 +331,50 @@ struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperatio
 
 struct CopyFileObjectStorageOperation final : public IDiskObjectStorageOperation
 {
+    /// Local paths
     std::string from_path;
     std::string to_path;
-    std::string remote_fs_root_path;
 
-    std::vector<std::string> created_blobs;
+    StoredObjects created_objects;
 
     CopyFileObjectStorageOperation(
         IObjectStorage & object_storage_,
         IMetadataStorage & metadata_storage_,
         const std::string & from_path_,
-        const std::string & to_path_,
-        const std::string & remote_fs_root_path_)
+        const std::string & to_path_)
         : IDiskObjectStorageOperation(object_storage_, metadata_storage_)
         , from_path(from_path_)
         , to_path(to_path_)
-        , remote_fs_root_path(remote_fs_root_path_)
     {}
 
+    std::string getInfoForLog() const override
+    {
+        return fmt::format("CopyFileObjectStorageOperation (path_from: {}, path_to: {})", from_path, to_path);
+    }
+
     void execute(MetadataTransactionPtr tx) override
     {
         tx->createEmptyMetadataFile(to_path);
-        auto source_blobs = metadata_storage.getObjectStoragePaths(from_path); /// Full paths
+        auto source_blobs = metadata_storage.getStorageObjects(from_path); /// Full paths
 
-        for (const auto & [blob_from, size] : source_blobs)
+        for (const auto & object_from : source_blobs)
         {
-            auto blob_name = getRandomASCIIString();
+            std::string blob_name = object_storage.generateBlobNameForPath(to_path);
+            auto object_to = StoredObject::create(
+                object_storage, fs::path(metadata_storage.getObjectStorageRootPath()) / blob_name);
 
-            auto blob_to = fs::path(remote_fs_root_path) / blob_name;
+            object_storage.copyObject(object_from, object_to);
 
-            object_storage.copyObject(blob_from, blob_to);
+            tx->addBlobToMetadata(to_path, blob_name, object_from.bytes_size);
 
-            tx->addBlobToMetadata(to_path, blob_name, size);
-
-            created_blobs.push_back(blob_to);
+            created_objects.push_back(object_to);
         }
     }
 
     void undo() override
     {
-        for (const auto & blob_path : created_blobs)
-            object_storage.removeObject(blob_path);
+        for (const auto & object : created_objects)
+            object_storage.removeObject(object);
     }
 
     void finalize() override
@@ -380,7 +398,7 @@ void DiskObjectStorageTransaction::createDirectories(const std::string & path)
     operations_to_execute.emplace_back(
         std::make_unique<PureMetadataObjectStorageOperation>(object_storage, metadata_storage, [path](MetadataTransactionPtr tx)
         {
-            tx->createDicrectoryRecursive(path);
+            tx->createDirectoryRecursive(path);
         }));
 }
 
@@ -411,7 +429,8 @@ void DiskObjectStorageTransaction::moveFile(const String & from_path, const Stri
 
 void DiskObjectStorageTransaction::replaceFile(const std::string & from_path, const std::string & to_path)
 {
-    operations_to_execute.emplace_back(std::make_unique<ReplaceFileObjectStorageOperation>(object_storage, metadata_storage, from_path, to_path));
+    auto operation = std::make_unique<ReplaceFileObjectStorageOperation>(object_storage, metadata_storage, from_path, to_path);
+    operations_to_execute.emplace_back(std::move(operation));
 }
 
 void DiskObjectStorageTransaction::clearDirectory(const std::string & path)
@@ -430,17 +449,22 @@ void DiskObjectStorageTransaction::removeFile(const std::string & path)
 
 void DiskObjectStorageTransaction::removeSharedFile(const std::string & path, bool keep_shared_data)
 {
-    operations_to_execute.emplace_back(std::make_unique<RemoveObjectStorageOperation>(object_storage, metadata_storage, path, keep_shared_data, false));
+    auto operation = std::make_unique<RemoveObjectStorageOperation>(object_storage, metadata_storage, path, keep_shared_data, false);
+    operations_to_execute.emplace_back(std::move(operation));
 }
 
-void DiskObjectStorageTransaction::removeSharedRecursive(const std::string & path, bool keep_all_shared_data, const NameSet & file_names_remove_metadata_only)
+void DiskObjectStorageTransaction::removeSharedRecursive(
+    const std::string & path, bool keep_all_shared_data, const NameSet & file_names_remove_metadata_only)
 {
-    operations_to_execute.emplace_back(std::make_unique<RemoveRecursiveObjectStorageOperation>(object_storage, metadata_storage, path, keep_all_shared_data, file_names_remove_metadata_only));
+    auto operation = std::make_unique<RemoveRecursiveObjectStorageOperation>(
+        object_storage, metadata_storage, path, keep_all_shared_data, file_names_remove_metadata_only);
+    operations_to_execute.emplace_back(std::move(operation));
 }
 
 void DiskObjectStorageTransaction::removeSharedFileIfExists(const std::string & path, bool keep_shared_data)
 {
-    operations_to_execute.emplace_back(std::make_unique<RemoveObjectStorageOperation>(object_storage, metadata_storage, path, keep_shared_data, true));
+    auto operation = std::make_unique<RemoveObjectStorageOperation>(object_storage, metadata_storage, path, keep_shared_data, true);
+    operations_to_execute.emplace_back(std::move(operation));
 }
 
 void DiskObjectStorageTransaction::removeDirectory(const std::string & path)
@@ -464,7 +488,8 @@ void DiskObjectStorageTransaction::removeFileIfExists(const std::string & path)
 }
 
 
-void DiskObjectStorageTransaction::removeSharedFiles(const RemoveBatchRequest & files, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only)
+void DiskObjectStorageTransaction::removeSharedFiles(
+    const RemoveBatchRequest & files, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only)
 {
     for (const auto & file : files)
     {
@@ -493,9 +518,10 @@ std::unique_ptr<WriteBufferFromFileBase> DiskObjectStorageTransaction::writeFile
     const WriteSettings & settings,
     bool autocommit)
 {
-    auto blob_name = getRandomASCIIString();
-
+    String blob_name;
     std::optional<ObjectAttributes> object_attributes;
+
+    blob_name = object_storage.generateBlobNameForPath(path);
     if (metadata_helper)
     {
         auto revision = metadata_helper->revision_counter + 1;
@@ -506,9 +532,8 @@ std::unique_ptr<WriteBufferFromFileBase> DiskObjectStorageTransaction::writeFile
         blob_name = "r" + revisionToString(revision) + "-file-" + blob_name;
     }
 
-    auto blob_path = fs::path(remote_fs_root_path) / blob_name;
-
-    auto write_operation = std::make_unique<WriteFileObjectStorageOperation>(object_storage, metadata_storage, path, blob_path);
+    auto object = StoredObject::create(object_storage, fs::path(metadata_storage.getObjectStorageRootPath()) / blob_name);
+    auto write_operation = std::make_unique<WriteFileObjectStorageOperation>(object_storage, metadata_storage, object);
     std::function<void(size_t count)> create_metadata_callback;
 
     if  (autocommit)
@@ -552,9 +577,12 @@ std::unique_ptr<WriteBufferFromFileBase> DiskObjectStorageTransaction::writeFile
 
     /// We always use mode Rewrite because we simulate append using metadata and different files
     return object_storage.writeObject(
-        blob_path, WriteMode::Rewrite, object_attributes,
+        object,
+        WriteMode::Rewrite,
+        object_attributes,
         std::move(create_metadata_callback),
-        buf_size, settings);
+        buf_size,
+        settings);
 }
 
 
@@ -596,7 +624,8 @@ void DiskObjectStorageTransaction::createFile(const std::string & path)
 
 void DiskObjectStorageTransaction::copyFile(const std::string & from_file_path, const std::string & to_file_path)
 {
-    operations_to_execute.emplace_back(std::make_unique<CopyFileObjectStorageOperation>(object_storage, metadata_storage, from_file_path, to_file_path, remote_fs_root_path));
+    operations_to_execute.emplace_back(
+        std::make_unique<CopyFileObjectStorageOperation>(object_storage, metadata_storage, from_file_path, to_file_path));
 }
 
 void DiskObjectStorageTransaction::commit()
@@ -609,7 +638,7 @@ void DiskObjectStorageTransaction::commit()
         }
         catch (Exception & ex)
         {
-            ex.addMessage(fmt::format("While executing operation #{}", i));
+            ex.addMessage(fmt::format("While executing operation #{} ({})", i, operations_to_execute[i]->getInfoForLog()));
 
             for (int64_t j = i; j >= 0; --j)
             {

src/Disks/ObjectStorages/DiskObjectStorageTransaction.h

@@ -30,12 +30,15 @@ public:
     /// like removal of blobs. Such implementation can lead to garbage.
     virtual void finalize() = 0;
     virtual ~IDiskObjectStorageOperation() = default;
+
+    virtual std::string getInfoForLog() const = 0;
 };
 
 using DiskObjectStorageOperation = std::unique_ptr<IDiskObjectStorageOperation>;
 
 using DiskObjectStorageOperations = std::vector<DiskObjectStorageOperation>;
 
+
 /// Disk object storage transaction, actually implement some part of disk object storage
 /// logic. Works on top of non atomic operations with blobs and possibly atomic implementation
 /// of metadata storage.
@@ -52,17 +55,18 @@ struct DiskObjectStorageTransaction final : public IDiskTransaction, std::enable
 private:
     IObjectStorage & object_storage;
     IMetadataStorage & metadata_storage;
+
     MetadataTransactionPtr metadata_transaction;
+
     /// TODO we can get rid of this params
-    const std::string & remote_fs_root_path;
     DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper;
 
     DiskObjectStorageOperations operations_to_execute;
+
 public:
     DiskObjectStorageTransaction(
         IObjectStorage & object_storage_,
         IMetadataStorage & metadata_storage_,
-        const std::string & remote_fs_root_path_,
         DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_);
 
     void commit() override;

src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.cpp

@@ -0,0 +1,280 @@
+#include "FakeMetadataStorageFromDisk.h"
+#include <Disks/IDisk.h>
+#include <Common/filesystemHelpers.h>
+#include <Common/logger_useful.h>
+#include <IO/WriteHelpers.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int NOT_IMPLEMENTED;
+    extern const int FS_METADATA_ERROR;
+}
+
+FakeMetadataStorageFromDisk::FakeMetadataStorageFromDisk(
+    DiskPtr disk_,
+    ObjectStoragePtr object_storage_,
+    const std::string & object_storage_root_path_)
+    : disk(disk_)
+    , object_storage(object_storage_)
+    , object_storage_root_path(object_storage_root_path_)
+{
+}
+
+MetadataTransactionPtr FakeMetadataStorageFromDisk::createTransaction() const
+{
+    return std::make_shared<FakeMetadataStorageFromDiskTransaction>(*this, disk);
+}
+
+const std::string & FakeMetadataStorageFromDisk::getPath() const
+{
+    return disk->getPath();
+}
+
+bool FakeMetadataStorageFromDisk::exists(const std::string & path) const
+{
+    return disk->exists(path);
+}
+
+bool FakeMetadataStorageFromDisk::isFile(const std::string & path) const
+{
+    return disk->isFile(path);
+}
+
+bool FakeMetadataStorageFromDisk::isDirectory(const std::string & path) const
+{
+    return disk->isDirectory(path);
+}
+
+Poco::Timestamp FakeMetadataStorageFromDisk::getLastModified(const std::string & path) const
+{
+    return disk->getLastModified(path);
+}
+
+time_t FakeMetadataStorageFromDisk::getLastChanged(const std::string & path) const
+{
+    return disk->getLastChanged(path);
+}
+
+uint64_t FakeMetadataStorageFromDisk::getFileSize(const String & path) const
+{
+    return disk->getFileSize(path);
+}
+
+std::vector<std::string> FakeMetadataStorageFromDisk::listDirectory(const std::string & path) const
+{
+    std::vector<std::string> result;
+    auto it = disk->iterateDirectory(path);
+    while (it->isValid())
+    {
+        result.push_back(it->path());
+        it->next();
+    }
+    return result;
+}
+
+DirectoryIteratorPtr FakeMetadataStorageFromDisk::iterateDirectory(const std::string & path) const
+{
+    return disk->iterateDirectory(path);
+}
+
+std::string FakeMetadataStorageFromDisk::readFileToString(const std::string &) const
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "readFileToString is not implemented for FakeMetadataStorageFromDisk");
+}
+
+std::unordered_map<String, String> FakeMetadataStorageFromDisk::getSerializedMetadata(const std::vector<String> &) const
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getSerializedMetadata is not implemented for FakeMetadataStorageFromDisk");
+}
+
+StoredObjects FakeMetadataStorageFromDisk::getStorageObjects(const std::string & path) const
+{
+    std::string blob_name = object_storage->generateBlobNameForPath(path);
+
+    std::string object_path = fs::path(object_storage_root_path) / blob_name;
+    size_t object_size = getFileSize(object_path);
+
+    auto object = StoredObject::create(*object_storage, object_path, object_size);
+    return {std::move(object)};
+}
+
+uint32_t FakeMetadataStorageFromDisk::getHardlinkCount(const std::string & path) const
+{
+    size_t ref_count = disk->getRefCount(path);
+    assert(ref_count > 0);
+    return ref_count - 1;
+}
+
+const IMetadataStorage & FakeMetadataStorageFromDiskTransaction::getStorageForNonTransactionalReads() const
+{
+    return metadata_storage;
+}
+
+void FakeMetadataStorageFromDiskTransaction::addOperation(MetadataOperationPtr && operation)
+{
+    if (state != MetadataFromDiskTransactionState::PREPARING)
+        throw Exception(
+            ErrorCodes::FS_METADATA_ERROR,
+            "Cannot add operations to transaction in {} state, it should be in {} state",
+            toString(state), toString(MetadataFromDiskTransactionState::PREPARING));
+
+    operations.emplace_back(std::move(operation));
+}
+
+void FakeMetadataStorageFromDiskTransaction::commit()
+{
+    if (state != MetadataFromDiskTransactionState::PREPARING)
+        throw Exception(
+            ErrorCodes::FS_METADATA_ERROR,
+            "Cannot commit transaction in {} state, it should be in {} state",
+            toString(state), toString(MetadataFromDiskTransactionState::PREPARING));
+
+    {
+        std::unique_lock lock(metadata_storage.metadata_mutex);
+        for (size_t i = 0; i < operations.size(); ++i)
+        {
+            try
+            {
+                operations[i]->execute();
+            }
+            catch (Exception & ex)
+            {
+                tryLogCurrentException(__PRETTY_FUNCTION__);
+                ex.addMessage(fmt::format("While committing metadata operation #{}", i));
+                state = MetadataFromDiskTransactionState::FAILED;
+                rollback(i);
+                throw;
+            }
+        }
+    }
+
+    /// Do it in "best effort" mode
+    for (size_t i = 0; i < operations.size(); ++i)
+    {
+        try
+        {
+            operations[i]->finalize();
+        }
+        catch (...)
+        {
+            tryLogCurrentException(__PRETTY_FUNCTION__, fmt::format("Failed to finalize operation #{}", i));
+        }
+    }
+
+    state = MetadataFromDiskTransactionState::COMMITTED;
+}
+
+void FakeMetadataStorageFromDiskTransaction::rollback(size_t until_pos)
+{
+    /// Otherwise everything is alright
+    if (state == MetadataFromDiskTransactionState::FAILED)
+    {
+        for (int64_t i = until_pos; i >= 0; --i)
+        {
+            try
+            {
+                operations[i]->undo();
+            }
+            catch (Exception & ex)
+            {
+                state = MetadataFromDiskTransactionState::PARTIALLY_ROLLED_BACK;
+                ex.addMessage(fmt::format("While rolling back operation #{}", i));
+                throw;
+            }
+        }
+    }
+    else
+    {
+        /// Nothing to do, transaction committed or not even started to commit
+    }
+}
+
+void FakeMetadataStorageFromDiskTransaction::writeStringToFile(const std::string & path, const std::string & data)
+{
+    auto wb = disk->writeFile(path);
+    wb->write(data.data(), data.size());
+    wb->finalize();
+}
+
+void FakeMetadataStorageFromDiskTransaction::setLastModified(const std::string & path, const Poco::Timestamp & timestamp)
+{
+    disk->setLastModified(path, timestamp);
+}
+
+void FakeMetadataStorageFromDiskTransaction::unlinkFile(const std::string & path)
+{
+    disk->removeFile(path);
+}
+
+void FakeMetadataStorageFromDiskTransaction::removeRecursive(const std::string & path)
+{
+    disk->removeRecursive(path);
+}
+
+void FakeMetadataStorageFromDiskTransaction::createDirectory(const std::string & path)
+{
+    disk->createDirectory(path);
+}
+
+void FakeMetadataStorageFromDiskTransaction::createDirectoryRecursive(const std::string & path)
+{
+    disk->createDirectories(path);
+}
+
+void FakeMetadataStorageFromDiskTransaction::removeDirectory(const std::string & path)
+{
+    disk->removeDirectory(path);
+}
+
+void FakeMetadataStorageFromDiskTransaction::moveFile(const std::string & path_from, const std::string & path_to)
+{
+    disk->moveFile(path_from, path_to);
+}
+
+void FakeMetadataStorageFromDiskTransaction::moveDirectory(const std::string & path_from, const std::string & path_to)
+{
+    disk->moveDirectory(path_from, path_to);
+}
+
+void FakeMetadataStorageFromDiskTransaction::replaceFile(const std::string & path_from, const std::string & path_to)
+{
+    disk->replaceFile(path_from, path_to);
+}
+
+void FakeMetadataStorageFromDiskTransaction::setReadOnly(const std::string & path)
+{
+    disk->setReadOnly(path);
+}
+
+void FakeMetadataStorageFromDiskTransaction::createHardLink(const std::string & path_from, const std::string & path_to)
+{
+    disk->createHardLink(path_from, path_to);
+}
+
+void FakeMetadataStorageFromDiskTransaction::createEmptyMetadataFile(const std::string & /* path */)
+{
+    /// Noop.
+}
+
+void FakeMetadataStorageFromDiskTransaction::createMetadataFile(
+    const std::string & /* path */, const std::string & /* blob_name */, uint64_t /* size_in_bytes */)
+{
+    /// Noop.
+}
+
+void FakeMetadataStorageFromDiskTransaction::addBlobToMetadata(
+    const std::string & /* path */, const std::string & /* blob_name */, uint64_t /* size_in_bytes */)
+{
+    /// Noop, local metadata files is only one file, it is the metadata file itself.
+}
+
+void FakeMetadataStorageFromDiskTransaction::unlinkMetadata(const std::string & path)
+{
+    disk->removeFile(path);
+}
+
+}

src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.h

@@ -0,0 +1,120 @@
+#pragma once
+
+#include <Disks/ObjectStorages/IMetadataStorage.h>
+#include <Disks/ObjectStorages/MetadataFromDiskTransactionState.h>
+#include <Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.h>
+
+
+namespace DB
+{
+
+class FakeMetadataStorageFromDisk final : public IMetadataStorage
+{
+private:
+    friend class FakeMetadataStorageFromDiskTransaction;
+
+    mutable std::shared_mutex metadata_mutex;
+
+    DiskPtr disk;
+    ObjectStoragePtr object_storage;
+    std::string object_storage_root_path;
+
+public:
+    FakeMetadataStorageFromDisk(
+        DiskPtr disk_,
+        ObjectStoragePtr object_storage_,
+        const std::string & object_storage_root_path_);
+
+    MetadataTransactionPtr createTransaction() const override;
+
+    const std::string & getPath() const override;
+
+    bool exists(const std::string & path) const override;
+
+    bool isFile(const std::string & path) const override;
+
+    bool isDirectory(const std::string & path) const override;
+
+    uint64_t getFileSize(const String & path) const override;
+
+    Poco::Timestamp getLastModified(const std::string & path) const override;
+
+    time_t getLastChanged(const std::string & path) const override;
+
+    std::vector<std::string> listDirectory(const std::string & path) const override;
+
+    DirectoryIteratorPtr iterateDirectory(const std::string & path) const override;
+
+    std::string readFileToString(const std::string & path) const override;
+
+    std::unordered_map<String, String> getSerializedMetadata(const std::vector<String> & file_paths) const override;
+
+    uint32_t getHardlinkCount(const std::string & path) const override;
+
+    DiskPtr getDisk() const { return disk; }
+
+    StoredObjects getStorageObjects(const std::string & path) const override;
+
+    std::string getObjectStorageRootPath() const override { return object_storage_root_path; }
+};
+
+class FakeMetadataStorageFromDiskTransaction final : public IMetadataTransaction
+{
+private:
+    DiskPtr disk;
+    const FakeMetadataStorageFromDisk & metadata_storage;
+
+    std::vector<MetadataOperationPtr> operations;
+    MetadataFromDiskTransactionState state{MetadataFromDiskTransactionState::PREPARING};
+
+    void addOperation(MetadataOperationPtr && operation);
+
+    void rollback(size_t until_pos);
+
+public:
+    FakeMetadataStorageFromDiskTransaction(
+        const FakeMetadataStorageFromDisk & metadata_storage_, DiskPtr disk_)
+        : disk(disk_)
+        , metadata_storage(metadata_storage_)
+    {}
+
+    ~FakeMetadataStorageFromDiskTransaction() override = default;
+
+    const IMetadataStorage & getStorageForNonTransactionalReads() const final;
+
+    void commit() final;
+
+    void writeStringToFile(const std::string & path, const std::string & data) override;
+
+    void createEmptyMetadataFile(const std::string & path) override;
+
+    void createMetadataFile(const std::string & path, const std::string & blob_name, uint64_t size_in_bytes) override;
+
+    void addBlobToMetadata(const std::string & path, const std::string & blob_name, uint64_t size_in_bytes) override;
+
+    void setLastModified(const std::string & path, const Poco::Timestamp & timestamp) override;
+
+    void setReadOnly(const std::string & path) override;
+
+    void unlinkFile(const std::string & path) override;
+
+    void createDirectory(const std::string & path) override;
+
+    void createDirectoryRecursive(const std::string & path) override;
+
+    void removeDirectory(const std::string & path) override;
+
+    void removeRecursive(const std::string & path) override;
+
+    void createHardLink(const std::string & path_from, const std::string & path_to) override;
+
+    void moveFile(const std::string & path_from, const std::string & path_to) override;
+
+    void moveDirectory(const std::string & path_from, const std::string & path_to) override;
+
+    void replaceFile(const std::string & path_from, const std::string & path_to) override;
+
+    void unlinkMetadata(const std::string & path) override;
+};
+
+}

src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp

@@ -11,6 +11,7 @@
 #include <Disks/IO/ReadIndirectBufferFromRemoteFS.h>
 #include <Disks/IO/WriteIndirectBufferFromRemoteFS.h>
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
+#include <Common/getRandomASCIIString.h>
 
 
 #if USE_HDFS
@@ -32,40 +33,46 @@ void HDFSObjectStorage::startup()
 {
 }
 
-bool HDFSObjectStorage::exists(const std::string & hdfs_uri) const
+std::string HDFSObjectStorage::generateBlobNameForPath(const std::string & /* path */)
 {
-    const size_t begin_of_path = hdfs_uri.find('/', hdfs_uri.find("//") + 2);
-    const String remote_fs_object_path = hdfs_uri.substr(begin_of_path);
+    return getRandomASCIIString();
+}
+
+bool HDFSObjectStorage::exists(const StoredObject & object) const
+{
+    const auto & path = object.absolute_path;
+    const size_t begin_of_path = path.find('/', path.find("//") + 2);
+    const String remote_fs_object_path = path.substr(begin_of_path);
     return (0 == hdfsExists(hdfs_fs.get(), remote_fs_object_path.c_str()));
 }
 
-std::unique_ptr<SeekableReadBuffer> HDFSObjectStorage::readObject( /// NOLINT
-    const std::string & path,
+std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObject( /// NOLINT
+    const StoredObject & object,
     const ReadSettings & read_settings,
     std::optional<size_t>,
     std::optional<size_t>) const
 {
-    return std::make_unique<ReadBufferFromHDFS>(path, path, config, read_settings.remote_fs_buffer_size);
+    return std::make_unique<ReadBufferFromHDFS>(object.absolute_path, object.absolute_path, config, patchSettings(read_settings));
 }
 
 std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObjects( /// NOLINT
-    const PathsWithSize & paths_to_read,
+    const StoredObjects & objects,
     const ReadSettings & read_settings,
     std::optional<size_t>,
     std::optional<size_t>) const
 {
-    auto hdfs_impl = std::make_unique<ReadBufferFromHDFSGather>(config, paths_to_read, read_settings);
+    auto hdfs_impl = std::make_unique<ReadBufferFromHDFSGather>(config, objects, patchSettings(read_settings));
     auto buf = std::make_unique<ReadIndirectBufferFromRemoteFS>(std::move(hdfs_impl));
     return std::make_unique<SeekAvoidingReadBuffer>(std::move(buf), settings->min_bytes_for_seek);
 }
 
 std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOLINT
-    const std::string & path,
+    const StoredObject & object,
     WriteMode mode,
     std::optional<ObjectAttributes> attributes,
     FinalizeCallback && finalize_callback,
     size_t buf_size,
-    const WriteSettings &)
+    const WriteSettings & write_settings)
 {
     if (attributes.has_value())
         throw Exception(
@@ -74,10 +81,10 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
 
     /// Single O_WRONLY in libhdfs adds O_TRUNC
     auto hdfs_buffer = std::make_unique<WriteBufferFromHDFS>(
-        path, config, settings->replication, buf_size,
+        object.absolute_path, config, settings->replication, patchSettings(write_settings), buf_size,
         mode == WriteMode::Rewrite ? O_WRONLY : O_WRONLY | O_APPEND);
 
-    return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(hdfs_buffer), std::move(finalize_callback), path);
+    return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(hdfs_buffer), std::move(finalize_callback), object.absolute_path);
 }
 
 
@@ -94,8 +101,9 @@ void HDFSObjectStorage::listPrefix(const std::string & path, RelativePathsWithSi
 }
 
 /// Remove file. Throws exception if file doesn't exists or it's a directory.
-void HDFSObjectStorage::removeObject(const std::string & path)
+void HDFSObjectStorage::removeObject(const StoredObject & object)
 {
+    const auto & path = object.absolute_path;
     const size_t begin_of_path = path.find('/', path.find("//") + 2);
 
     /// Add path from root to file name
@@ -105,22 +113,22 @@ void HDFSObjectStorage::removeObject(const std::string & path)
 
 }
 
-void HDFSObjectStorage::removeObjects(const PathsWithSize & paths)
+void HDFSObjectStorage::removeObjects(const StoredObjects & objects)
 {
-    for (const auto & [path, _] : paths)
-        removeObject(path);
+    for (const auto & object : objects)
+        removeObject(object);
 }
 
-void HDFSObjectStorage::removeObjectIfExists(const std::string & path)
+void HDFSObjectStorage::removeObjectIfExists(const StoredObject & object)
 {
-    if (exists(path))
-        removeObject(path);
+    if (exists(object))
+        removeObject(object);
 }
 
-void HDFSObjectStorage::removeObjectsIfExist(const PathsWithSize & paths)
+void HDFSObjectStorage::removeObjectsIfExist(const StoredObjects & objects)
 {
-    for (const auto & [path, _] : paths)
-        removeObjectIfExists(path);
+    for (const auto & object : objects)
+        removeObjectIfExists(object);
 }
 
 ObjectMetadata HDFSObjectStorage::getObjectMetadata(const std::string &) const
@@ -131,8 +139,8 @@ ObjectMetadata HDFSObjectStorage::getObjectMetadata(const std::string &) const
 }
 
 void HDFSObjectStorage::copyObject( /// NOLINT
-    const std::string & object_from,
-    const std::string & object_to,
+    const StoredObject & object_from,
+    const StoredObject & object_to,
     std::optional<ObjectAttributes> object_to_attributes)
 {
     if (object_to_attributes.has_value())
@@ -147,8 +155,9 @@ void HDFSObjectStorage::copyObject( /// NOLINT
 }
 
 
-void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr)
+void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr context)
 {
+    applyRemoteThrottlingSettings(context);
 }
 
 std::unique_ptr<IObjectStorage> HDFSObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr)

src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h

@@ -41,34 +41,34 @@ public:
     using SettingsPtr = std::unique_ptr<HDFSObjectStorageSettings>;
 
     HDFSObjectStorage(
-        FileCachePtr && cache_,
         const String & hdfs_root_path_,
         SettingsPtr settings_,
         const Poco::Util::AbstractConfiguration & config_)
-        : IObjectStorage(std::move(cache_))
-        , config(config_)
+        : config(config_)
         , hdfs_builder(createHDFSBuilder(hdfs_root_path_, config))
         , hdfs_fs(createHDFSFS(hdfs_builder.get()))
         , settings(std::move(settings_))
     {}
 
-    bool exists(const std::string & hdfs_uri) const override;
+    std::string getName() const override { return "HDFSObjectStorage"; }
 
-    std::unique_ptr<SeekableReadBuffer> readObject( /// NOLINT
-        const std::string & path,
+    bool exists(const StoredObject & object) const override;
+
+    std::unique_ptr<ReadBufferFromFileBase> readObject( /// NOLINT
+        const StoredObject & object,
         const ReadSettings & read_settings = ReadSettings{},
         std::optional<size_t> read_hint = {},
         std::optional<size_t> file_size = {}) const override;
 
     std::unique_ptr<ReadBufferFromFileBase> readObjects( /// NOLINT
-        const PathsWithSize & paths_to_read,
+        const StoredObjects & objects,
         const ReadSettings & read_settings = ReadSettings{},
         std::optional<size_t> read_hint = {},
         std::optional<size_t> file_size = {}) const override;
 
     /// Open the file for write and return WriteBufferFromFileBase object.
     std::unique_ptr<WriteBufferFromFileBase> writeObject( /// NOLINT
-        const std::string & path,
+        const StoredObject & object,
         WriteMode mode,
         std::optional<ObjectAttributes> attributes = {},
         FinalizeCallback && finalize_callback = {},
@@ -78,19 +78,19 @@ public:
     void listPrefix(const std::string & path, RelativePathsWithSize & children) const override;
 
     /// Remove file. Throws exception if file doesn't exists or it's a directory.
-    void removeObject(const std::string & path) override;
+    void removeObject(const StoredObject & object) override;
 
-    void removeObjects(const PathsWithSize & paths) override;
+    void removeObjects(const StoredObjects & objects) override;
 
-    void removeObjectIfExists(const std::string & path) override;
+    void removeObjectIfExists(const StoredObject & object) override;
 
-    void removeObjectsIfExist(const PathsWithSize & paths) override;
+    void removeObjectsIfExist(const StoredObjects & objects) override;
 
     ObjectMetadata getObjectMetadata(const std::string & path) const override;
 
     void copyObject( /// NOLINT
-        const std::string & object_from,
-        const std::string & object_to,
+        const StoredObject & object_from,
+        const StoredObject & object_to,
         std::optional<ObjectAttributes> object_to_attributes = {}) override;
 
     void shutdown() override;
@@ -110,6 +110,10 @@ public:
         const std::string & config_prefix,
         ContextPtr context) override;
 
+    std::string generateBlobNameForPath(const std::string & path) override;
+
+    bool isRemote() const override { return true; }
+
 private:
     const Poco::Util::AbstractConfiguration & config;
 

src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp

@@ -36,7 +36,7 @@ void registerDiskHDFS(DiskFactory & factory)
 
 
         /// FIXME Cache currently unsupported :(
-        ObjectStoragePtr hdfs_storage = std::make_unique<HDFSObjectStorage>(nullptr, uri, std::move(settings), config);
+        ObjectStoragePtr hdfs_storage = std::make_unique<HDFSObjectStorage>(uri, std::move(settings), config);
 
         auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context_);
 

src/Disks/ObjectStorages/IMetadataStorage.h

@@ -23,7 +23,7 @@ class IMetadataStorage;
 /// interface. This transaction is more like "batch operation" than real "transaction".
 ///
 /// But for better usability we can get MetadataStorage interface and use some read methods.
-struct IMetadataTransaction : private boost::noncopyable
+class IMetadataTransaction : private boost::noncopyable
 {
 public:
     virtual void commit() = 0;
@@ -43,7 +43,7 @@ public:
 
     virtual void createDirectory(const std::string & path) = 0;
 
-    virtual void createDicrectoryRecursive(const std::string & path) = 0;
+    virtual void createDirectoryRecursive(const std::string & path) = 0;
 
     virtual void removeDirectory(const std::string & path) = 0;
 
@@ -85,12 +85,16 @@ using MetadataTransactionPtr = std::shared_ptr<IMetadataTransaction>;
 /// small amounts of data (strings).
 class IMetadataStorage : private boost::noncopyable
 {
+friend class MetadataStorageFromDiskTransaction;
+
 public:
     virtual MetadataTransactionPtr createTransaction() const = 0;
 
-    /// General purpose functions (similar to Disk)
+    /// Get metadata root path.
     virtual const std::string & getPath() const = 0;
 
+    /// ==== General purpose methods. Define properties of object storage file based on metadata files ====
+
     virtual bool exists(const std::string & path) const = 0;
 
     virtual bool isFile(const std::string & path) const = 0;
@@ -119,9 +123,11 @@ public:
     /// Read multiple metadata files into strings and return mapping from file_path -> metadata
     virtual std::unordered_map<std::string, std::string> getSerializedMetadata(const std::vector<String> & file_paths) const = 0;
 
-    /// Return [(object_storage_path, size_in_bytes), ...] for metadata path
-    /// object_storage_path is a full path to the blob.
-    virtual PathsWithSize getObjectStoragePaths(const std::string & path) const = 0;
+    /// Return object information (absolute_path, bytes_size, ...) for metadata path.
+    /// object_storage_path is absolute.
+    virtual StoredObjects getStorageObjects(const std::string & path) const = 0;
+
+    virtual std::string getObjectStorageRootPath() const = 0;
 };
 
 using MetadataStoragePtr = std::shared_ptr<IMetadataStorage>;

src/Disks/ObjectStorages/IObjectStorage.cpp

@@ -6,6 +6,11 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int NOT_IMPLEMENTED;
+}
+
 AsynchronousReaderPtr IObjectStorage::getThreadPoolReader()
 {
     constexpr size_t pool_size = 50;
@@ -22,21 +27,11 @@ ThreadPool & IObjectStorage::getThreadPoolWriter()
     return writer;
 }
 
-std::string IObjectStorage::getCacheBasePath() const
-{
-    return cache ? cache->getBasePath() : "";
-}
-
-void IObjectStorage::removeFromCache(const std::string & path)
-{
-    if (cache)
-    {
-        auto key = cache->hash(path);
-        cache->removeIfExists(key);
-    }
-}
-
-void IObjectStorage::copyObjectToAnotherObjectStorage(const std::string & object_from, const std::string & object_to, IObjectStorage & object_storage_to, std::optional<ObjectAttributes> object_to_attributes) // NOLINT
+void IObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT
+    const StoredObject & object_from,
+    const StoredObject & object_to,
+    IObjectStorage & object_storage_to,
+    std::optional<ObjectAttributes> object_to_attributes)
 {
     if (&object_storage_to == this)
         copyObject(object_from, object_to, object_to_attributes);
@@ -47,4 +42,32 @@ void IObjectStorage::copyObjectToAnotherObjectStorage(const std::string & object
     out->finalize();
 }
 
+std::string IObjectStorage::getCacheBasePath() const
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getCacheBasePath() is not implemented for {}", getName());
+}
+
+void IObjectStorage::applyRemoteThrottlingSettings(ContextPtr context)
+{
+    std::unique_lock lock{throttlers_mutex};
+    remote_read_throttler = context->getRemoteReadThrottler();
+    remote_write_throttler = context->getRemoteWriteThrottler();
+}
+
+ReadSettings IObjectStorage::patchSettings(const ReadSettings & read_settings) const
+{
+    std::unique_lock lock{throttlers_mutex};
+    ReadSettings settings{read_settings};
+    settings.remote_throttler = remote_read_throttler;
+    return settings;
+}
+
+WriteSettings IObjectStorage::patchSettings(const WriteSettings & write_settings) const
+{
+    std::unique_lock lock{throttlers_mutex};
+    WriteSettings settings{write_settings};
+    settings.remote_throttler = remote_write_throttler;
+    return settings;
+}
+
 }