Merge branch 'master' into nullable-partition-fix1

.github/workflows/master.yml

@@ -2902,7 +2902,7 @@ jobs:
 #############################################################################################
 #################################### PERFORMANCE TESTS ######################################
 #############################################################################################
-  PerformanceComparison0:
+  PerformanceComparisonX86-0:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -2940,7 +2940,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison1:
+  PerformanceComparisonX86-1:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -2978,7 +2978,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison2:
+  PerformanceComparisonX86-2:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3016,7 +3016,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison3:
+  PerformanceComparisonX86-3:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3096,10 +3096,10 @@ jobs:
       - IntegrationTestsTsan1
       - IntegrationTestsTsan2
       - IntegrationTestsTsan3
-      - PerformanceComparison0
-      - PerformanceComparison1
-      - PerformanceComparison2
-      - PerformanceComparison3
+      - PerformanceComparisonX86-0
+      - PerformanceComparisonX86-1
+      - PerformanceComparisonX86-2
+      - PerformanceComparisonX86-3
       - CompatibilityCheck
       - ASTFuzzerTestDebug
       - ASTFuzzerTestAsan

.github/workflows/pull_request.yml

@@ -3118,7 +3118,7 @@ jobs:
 #############################################################################################
 #################################### PERFORMANCE TESTS ######################################
 #############################################################################################
-  PerformanceComparison0:
+  PerformanceComparisonX86-0:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3156,7 +3156,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison1:
+  PerformanceComparisonX86-1:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3194,7 +3194,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison2:
+  PerformanceComparisonX86-2:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3232,7 +3232,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparison3:
+  PerformanceComparisonX86-3:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, stress-tester]
     steps:
@@ -3270,7 +3270,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch0:
+  PerformanceComparisonAarch-0:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3308,7 +3308,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch1:
+  PerformanceComparisonAarch-1:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3346,7 +3346,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch2:
+  PerformanceComparisonAarch-2:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3384,7 +3384,7 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  PerformanceComparisonAarch3:
+  PerformanceComparisonAarch-3:
     needs: [BuilderDebAarch64]
     runs-on: [self-hosted, func-tester-aarch64]
     steps:
@@ -3481,14 +3481,14 @@ jobs:
       - IntegrationTestsTsan1
       - IntegrationTestsTsan2
       - IntegrationTestsTsan3
-      - PerformanceComparison0
-      - PerformanceComparison1
-      - PerformanceComparison2
-      - PerformanceComparison3
-      - PerformanceComparisonAarch0
-      - PerformanceComparisonAarch1
-      - PerformanceComparisonAarch2
-      - PerformanceComparisonAarch3
+      - PerformanceComparisonX86-0
+      - PerformanceComparisonX86-1
+      - PerformanceComparisonX86-2
+      - PerformanceComparisonX86-3
+      - PerformanceComparisonAarch-0
+      - PerformanceComparisonAarch-1
+      - PerformanceComparisonAarch-2
+      - PerformanceComparisonAarch-3
       - UnitTestsAsan
       - UnitTestsTsan
       - UnitTestsMsan

cmake/warnings.cmake

@@ -20,13 +20,9 @@ if (COMPILER_CLANG)
     # We want to get everything out of the compiler for code quality.
     add_warning(everything)
     add_warning(pedantic)
-    no_warning(vla-extension)
     no_warning(zero-length-array)
-    no_warning(c11-extensions)
-    no_warning(unused-command-line-argument)
     no_warning(c++98-compat-pedantic)
     no_warning(c++98-compat)
-    no_warning(c99-extensions)
     no_warning(conversion)
     no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
     no_warning(disabled-macro-expansion)
@@ -37,12 +33,7 @@ if (COMPILER_CLANG)
     no_warning(global-constructors)
     no_warning(missing-prototypes)
     no_warning(missing-variable-declarations)
-    no_warning(nested-anon-types)
-    no_warning(packed)
     no_warning(padded)
-    no_warning(return-std-move-in-c++11) # clang 7+
-    no_warning(shift-sign-overflow)
-    no_warning(sign-conversion)
     no_warning(switch-enum)
     no_warning(undefined-func-template)
     no_warning(unused-template)

contrib/thrift

@@ -1 +1 @@
-Subproject commit 010ccf0a0c7023fea0f6bf4e4078ebdff7e61982
+Subproject commit 2a93df80f27739ccabb5b885cb12a8dc7595ecdf

contrib/thrift-cmake/CMakeLists.txt

@@ -15,7 +15,6 @@ set(thriftcpp_SOURCES
         "${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.cpp"
         "${LIBRARY_DIR}/src/thrift/concurrency/ThreadManager.cpp"
         "${LIBRARY_DIR}/src/thrift/concurrency/TimerManager.cpp"
-        "${LIBRARY_DIR}/src/thrift/concurrency/Util.cpp"
         "${LIBRARY_DIR}/src/thrift/processor/PeekProcessor.cpp"
         "${LIBRARY_DIR}/src/thrift/protocol/TBase64Utils.cpp"
         "${LIBRARY_DIR}/src/thrift/protocol/TDebugProtocol.cpp"
@@ -33,6 +32,8 @@ set(thriftcpp_SOURCES
         "${LIBRARY_DIR}/src/thrift/transport/TServerSocket.cpp"
         "${LIBRARY_DIR}/src/thrift/transport/TTransportUtils.cpp"
         "${LIBRARY_DIR}/src/thrift/transport/TBufferTransports.cpp"
+        "${LIBRARY_DIR}/src/thrift/transport/SocketCommon.cpp"
+        "${LIBRARY_DIR}/src/thrift/transport/TWebSocketServer.cpp"
         "${LIBRARY_DIR}/src/thrift/server/TConnectedClient.cpp"
         "${LIBRARY_DIR}/src/thrift/server/TServerFramework.cpp"
         "${LIBRARY_DIR}/src/thrift/server/TSimpleServer.cpp"
@@ -92,4 +93,4 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 add_library(_thrift ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
 add_library(ch_contrib::thrift ALIAS _thrift)
 target_include_directories(_thrift SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src" ${CMAKE_CURRENT_BINARY_DIR})
-target_link_libraries (_thrift PUBLIC boost::headers_only)
+target_link_libraries (_thrift PUBLIC OpenSSL::SSL boost::headers_only)

docker/packager/binary/build.sh

@@ -29,17 +29,21 @@ env
 
 if [ -n "$MAKE_DEB" ]; then
   rm -rf /build/packages/root
-  if [ -z "$SANITIZER" ]; then
-    # We need to check if clickhouse-diagnostics is fine and build it
-    (
-      cd /build/programs/diagnostics
-      make test-no-docker
-      GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build
-      mv clickhouse-diagnostics ..
-    )
-  else
-    echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics
-    chmod +x /build/programs/clickhouse-diagnostics
+  # NOTE: this is for backward compatibility with previous releases,
+  # that does not diagnostics tool (only script).
+  if [ -d /build/programs/diagnostics ]; then
+    if [ -z "$SANITIZER" ]; then
+      # We need to check if clickhouse-diagnostics is fine and build it
+      (
+        cd /build/programs/diagnostics
+        make test-no-docker
+        GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build
+        mv clickhouse-diagnostics ..
+      )
+    else
+      echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics
+      chmod +x /build/programs/clickhouse-diagnostics
+    fi
   fi
 fi
 

docker/test/fasttest/Dockerfile

@@ -71,6 +71,7 @@ RUN apt-get update \
         software-properties-common \
         tzdata \
         unixodbc \
+        file \
        --yes --no-install-recommends
 
 RUN pip3 install numpy scipy pandas Jinja2

docker/test/stateless/Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update -y \
             unixodbc \
             wget \
             zstd \
+            file \
     && apt-get clean
 
 

docker/test/stress/run.sh

@@ -322,7 +322,11 @@ else
     clickhouse-client --query="SELECT 'Server version: ', version()"
 
     # Install new package before running stress test because we should use new clickhouse-client and new clickhouse-test
+    # But we should leave old binary in /usr/bin/ for gdb (so it will print sane stacktarces)
+    mv /usr/bin/clickhouse previous_release_package_folder/
     install_packages package_folder
+    mv /usr/bin/clickhouse package_folder/
+    mv previous_release_package_folder/clickhouse /usr/bin/
 
     mkdir tmp_stress_output
 
@@ -337,6 +341,7 @@ else
     mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
 
     # Start new server
+    mv package_folder/clickhouse /usr/bin/
     configure
     start 500
     clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
@@ -354,6 +359,9 @@ else
     # Error messages (we should ignore some errors)
     # FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.")
     # FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64")
+    # FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
+    #       Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
+    # FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
     echo "Check for Error messages in server log:"
     zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
                -e "Code: 236. DB::Exception: Cancelled mutating parts" \
@@ -378,6 +386,9 @@ else
                -e "is lost forever." \
                -e "Unknown index: idx." \
                -e "Cannot parse string 'Hello' as UInt64" \
+               -e "} <Error> TCPHandler: Code:" \
+               -e "} <Error> executeQuery: Code:" \
+               -e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
         /var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
         && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
         || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv

docs/en/operations/system-tables/part_log.md

@@ -19,6 +19,10 @@ The `system.part_log` table contains the following columns:
     -   `REGULAR_MERGE` — Some regular merge.
     -   `TTL_DELETE_MERGE` — Cleaning up expired data.
     -   `TTL_RECOMPRESS_MERGE` — Recompressing data part with the.
+-   `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Merge algorithm for the event with type `MERGE_PARTS`. Can have one of the following values:
+    -   `UNDECIDED`
+    -   `HORIZONTAL`
+    -   `VERTICAL`
 -   `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
 -   `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
 -   `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision.
@@ -52,6 +56,7 @@ Row 1:
 query_id:                      983ad9c7-28d5-4ae1-844e-603116b7de31
 event_type:                    NewPart
 merge_reason:                  NotAMerge
+merge_algorithm:               Undecided
 event_date:                    2021-02-02
 event_time:                    2021-02-02 11:14:28
 event_time_microseconds:                    2021-02-02 11:14:28.861919

docs/en/sql-reference/functions/string-search-functions.md

@@ -374,7 +374,7 @@ The same as `multiMatchAny`, but returns any index that matches the haystack.
 
 ## multiMatchAllIndices(haystack, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
 
-The same as `multiMatchAny`, but returns the array of all indicies that match the haystack in any order.
+The same as `multiMatchAny`, but returns the array of all indices that match the haystack in any order.
 
 ## multiFuzzyMatchAny(haystack, distance, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
 

docs/ru/operations/system-tables/part_log.md

@@ -14,6 +14,15 @@
     -   `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
     -   `MUTATE_PART` — изменение куска.
     -   `MOVE_PART` — перемещение куска между дисками.
+-   `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — Причина события с типом `MERGE_PARTS`. Может принимать одно из следующих значений:
+    -   `NOT_A_MERGE` — событие имеет тип иной, чем `MERGE_PARTS`.
+    -   `REGULAR_MERGE` — обычное слияние.
+    -   `TTL_DELETE_MERGE` — очистка истекших данных.
+    -   `TTL_RECOMPRESS_MERGE` — переупаковка куска.
+-   `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Алгоритм слияния для события с типом `MERGE_PARTS`. Может принимать одно из следующих значений:
+    -   `UNDECIDED`
+    -   `HORIZONTAL`
+    -   `VERTICAL`
 -   `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата события.
 -   `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время события.
 -   `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время события с точностью до микросекунд.
@@ -46,6 +55,8 @@ Row 1:
 ──────
 query_id:                      983ad9c7-28d5-4ae1-844e-603116b7de31
 event_type:                    NewPart
+merge_reason:                  NotAMerge
+merge_algorithm:               Undecided
 event_date:                    2021-02-02
 event_time:                    2021-02-02 11:14:28
 event_time_microseconds:                    2021-02-02 11:14:28.861919

programs/client/Client.cpp

@@ -106,7 +106,9 @@ void Client::processError(const String & query) const
 std::vector<String> Client::loadWarningMessages()
 {
     std::vector<String> messages;
-    connection->sendQuery(connection_parameters.timeouts, "SELECT message FROM system.warnings", "" /* query_id */,
+    connection->sendQuery(connection_parameters.timeouts,
+                          "SELECT * FROM viewIfPermitted(SELECT message FROM system.warnings ELSE null('message String'))",
+                          "" /* query_id */,
                           QueryProcessingStage::Complete,
                           &global_context->getSettingsRef(),
                           &global_context->getClientInfo(), false, {});

programs/copier/ZooKeeperStaff.h

@@ -20,7 +20,7 @@ public:
 
     bool operator<=(const WrappingUInt32 & other) const
     {
-        const UInt32 HALF = 1 << 31;
+        const UInt32 HALF = static_cast<UInt32>(1) << 31;
         return (value <= other.value && other.value - value < HALF)
                || (value > other.value && value - other.value > HALF);
     }

programs/server/config.xml

@@ -604,9 +604,23 @@
              if this setting is true the user B will see all rows, and if this setting is false the user B will see no rows.
              By default this setting is false for compatibility with earlier access configurations. -->
         <users_without_row_policies_can_read_rows>false</users_without_row_policies_can_read_rows>
+        
         <!-- By default, for backward compatibility ON CLUSTER queries ignore CLUSTER grant,
              however you can change this behaviour by setting this to true -->
         <on_cluster_queries_require_cluster_grant>false</on_cluster_queries_require_cluster_grant>
+
+        <!-- By default, for backward compatibility "SELECT * FROM system.<table>" doesn't require any grants and can be executed
+             by any user. You can change this behaviour by setting this to true.
+             If it's set to true then this query requires "GRANT SELECT ON system.<table>" just like as for non-system tables.
+             Exceptions: a few system tables ("tables", "columns", "databases", and some constant tables like "one", "contributors")
+             are still accessible for everyone; and if there is a SHOW privilege (e.g. "SHOW USERS") granted the corresponding system
+             table (i.e. "system.users") will be accessible. -->
+        <select_from_system_db_requires_grant>false</select_from_system_db_requires_grant>
+
+        <!-- By default, for backward compatibility "SELECT * FROM information_schema.<table>" doesn't require any grants and can be
+             executed by any user. You can change this behaviour by setting this to true.
+             If it's set to true then this query requires "GRANT SELECT ON information_schema.<table>" just like as for ordinary tables. -->
+        <select_from_information_schema_requires_grant>false</select_from_information_schema_requires_grant>
     </access_control_improvements>
 
     <!-- Default profile of settings. -->

src/Access/AccessControl.cpp

@@ -101,7 +101,7 @@ public:
         registered_prefixes = prefixes_;
     }
 
-    bool isSettingNameAllowed(const std::string_view & setting_name) const
+    bool isSettingNameAllowed(std::string_view setting_name) const
     {
         if (Settings::hasBuiltin(setting_name))
             return true;
@@ -116,7 +116,7 @@ public:
         return false;
     }
 
-    void checkSettingNameIsAllowed(const std::string_view & setting_name) const
+    void checkSettingNameIsAllowed(std::string_view setting_name) const
     {
         if (isSettingNameAllowed(setting_name))
             return;
@@ -165,13 +165,12 @@ void AccessControl::setUpFromMainConfig(const Poco::Util::AbstractConfiguration
     setNoPasswordAllowed(config_.getBool("allow_no_password", true));
     setPlaintextPasswordAllowed(config_.getBool("allow_plaintext_password", true));
 
-    setEnabledUsersWithoutRowPoliciesCanReadRows(config_.getBool(
-        "access_control_improvements.users_without_row_policies_can_read_rows",
-        false /* false because we need to be compatible with earlier access configurations */));
-
-    setOnClusterQueriesRequireClusterGrant(config_.getBool(
-        "access_control_improvements.on_cluster_queries_require_cluster_grant",
-        false /* false because we need to be compatible with earlier access configurations */));
+    /// Optional improvements in access control system.
+    /// The default values are false because we need to be compatible with earlier access configurations
+    setEnabledUsersWithoutRowPoliciesCanReadRows(config_.getBool("access_control_improvements.users_without_row_policies_can_read_rows", false));
+    setOnClusterQueriesRequireClusterGrant(config_.getBool("access_control_improvements.on_cluster_queries_require_cluster_grant", false));
+    setSelectFromSystemDatabaseRequiresGrant(config_.getBool("access_control_improvements.select_from_system_db_requires_grant", false));
+    setSelectFromInformationSchemaRequiresGrant(config_.getBool("access_control_improvements.select_from_information_schema_requires_grant", false));
 
     addStoragesFromMainConfig(config_, config_path_, get_zookeeper_function_);
 }

src/Access/AccessControl.h

@@ -152,6 +152,12 @@ public:
     void setOnClusterQueriesRequireClusterGrant(bool enable) { on_cluster_queries_require_cluster_grant = enable; }
     bool doesOnClusterQueriesRequireClusterGrant() const { return on_cluster_queries_require_cluster_grant; }
 
+    void setSelectFromSystemDatabaseRequiresGrant(bool enable) { select_from_system_db_requires_grant = enable; }
+    bool doesSelectFromSystemDatabaseRequireGrant() const { return select_from_system_db_requires_grant; }
+
+    void setSelectFromInformationSchemaRequiresGrant(bool enable) { select_from_information_schema_requires_grant = enable; }
+    bool doesSelectFromInformationSchemaRequireGrant() const { return select_from_information_schema_requires_grant; }
+
     std::shared_ptr<const ContextAccess> getContextAccess(
         const UUID & user_id,
         const std::vector<UUID> & current_roles,
@@ -215,6 +221,8 @@ private:
     std::atomic_bool allow_no_password = true;
     std::atomic_bool users_without_row_policies_can_read_rows = false;
     std::atomic_bool on_cluster_queries_require_cluster_grant = false;
+    std::atomic_bool select_from_system_db_requires_grant = false;
+    std::atomic_bool select_from_information_schema_requires_grant = false;
 };
 
 }

src/Access/AccessRights.cpp

@@ -252,7 +252,7 @@ public:
     }
 
     template <typename ... Args>
-    void grant(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
+    void grant(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
     {
         auto & child = getChild(name);
         child.grant(flags_, subnames...);
@@ -279,7 +279,7 @@ public:
     }
 
     template <typename... Args>
-    void revoke(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
+    void revoke(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
     {
         auto & child = getChild(name);
 
@@ -306,7 +306,7 @@ public:
     }
 
     template <typename... Args>
-    bool isGranted(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames) const
+    bool isGranted(const AccessFlags & flags_, std::string_view name, const Args &... subnames) const
     {
         AccessFlags flags_to_check = flags_ - min_flags_with_children;
         if (!flags_to_check)
@@ -388,11 +388,11 @@ public:
         return res;
     }
 
-    void modifyFlags(const ModifyFlagsFunction & function, bool & flags_added, bool & flags_removed)
+    void modifyFlags(const ModifyFlagsFunction & function, bool grant_option, bool & flags_added, bool & flags_removed)
     {
         flags_added = false;
         flags_removed = false;
-        modifyFlagsRec(function, flags_added, flags_removed);
+        modifyFlagsRec(function, grant_option, flags_added, flags_removed);
         if (flags_added || flags_removed)
             optimizeTree();
     }
@@ -415,7 +415,7 @@ private:
     AccessFlags getAllGrantableFlags() const { return ::DB::getAllGrantableFlags(level); }
     AccessFlags getChildAllGrantableFlags() const { return ::DB::getAllGrantableFlags(static_cast<Level>(level + 1)); }
 
-    Node * tryGetChild(const std::string_view & name) const
+    Node * tryGetChild(std::string_view name) const
     {
         if (!children)
             return nullptr;
@@ -425,7 +425,7 @@ private:
         return &it->second;
     }
 
-    Node & getChild(const std::string_view & name)
+    Node & getChild(std::string_view name)
     {
         auto * child = tryGetChild(name);
         if (child)
@@ -669,11 +669,11 @@ private:
     }
 
     template <typename ... ParentNames>
-    void modifyFlagsRec(const ModifyFlagsFunction & function, bool & flags_added, bool & flags_removed, const ParentNames & ... parent_names)
+    void modifyFlagsRec(const ModifyFlagsFunction & function, bool grant_option, bool & flags_added, bool & flags_removed, const ParentNames & ... parent_names)
     {
-        auto invoke = [&function](const AccessFlags & flags_, const AccessFlags & min_flags_with_children_, const AccessFlags & max_flags_with_children_, std::string_view database_ = {}, std::string_view table_ = {}, std::string_view column_ = {}) -> AccessFlags
+        auto invoke = [function, grant_option](const AccessFlags & flags_, const AccessFlags & min_flags_with_children_, const AccessFlags & max_flags_with_children_, std::string_view database_ = {}, std::string_view table_ = {}, std::string_view column_ = {}) -> AccessFlags
         {
-            return function(flags_, min_flags_with_children_, max_flags_with_children_, database_, table_, column_);
+            return function(flags_, min_flags_with_children_, max_flags_with_children_, database_, table_, column_, grant_option);
         };
 
         if constexpr (sizeof...(ParentNames) < 3)
@@ -683,7 +683,7 @@ private:
                 for (auto & child : *children | boost::adaptors::map_values)
                 {
                     const String & child_name = *child.node_name;
-                    child.modifyFlagsRec(function, flags_added, flags_removed, parent_names..., child_name);
+                    child.modifyFlagsRec(function, grant_option, flags_added, flags_removed, parent_names..., child_name);
                 }
             }
         }
@@ -819,20 +819,20 @@ void AccessRights::grantImpl(const AccessRightsElements & elements)
 }
 
 void AccessRights::grant(const AccessFlags & flags) { grantImpl<false>(flags); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl<false>(flags, database); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<false>(flags, database, table); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<false>(flags, database, table, column); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
-void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database) { grantImpl<false>(flags, database); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<false>(flags, database, table); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<false>(flags, database, table, column); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
+void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
 void AccessRights::grant(const AccessRightsElement & element) { grantImpl<false>(element); }
 void AccessRights::grant(const AccessRightsElements & elements) { grantImpl<false>(elements); }
 
 void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl<true>(flags); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database) { grantImpl<true>(flags, database); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<true>(flags, database, table); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<true>(flags, database, table, column); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
-void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database) { grantImpl<true>(flags, database); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<true>(flags, database, table); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<true>(flags, database, table, column); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
+void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
 void AccessRights::grantWithGrantOption(const AccessRightsElement & element) { grantImpl<true>(element); }
 void AccessRights::grantWithGrantOption(const AccessRightsElements & elements) { grantImpl<true>(elements); }
 
@@ -892,20 +892,20 @@ void AccessRights::revokeImpl(const AccessRightsElements & elements)
 }
 
 void AccessRights::revoke(const AccessFlags & flags) { revokeImpl<false>(flags); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl<false>(flags, database); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<false>(flags, database, table); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<false>(flags, database, table, column); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
-void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database) { revokeImpl<false>(flags, database); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<false>(flags, database, table); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<false>(flags, database, table, column); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
+void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
 void AccessRights::revoke(const AccessRightsElement & element) { revokeImpl<false>(element); }
 void AccessRights::revoke(const AccessRightsElements & elements) { revokeImpl<false>(elements); }
 
 void AccessRights::revokeGrantOption(const AccessFlags & flags) { revokeImpl<true>(flags); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database) { revokeImpl<true>(flags, database); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<true>(flags, database, table); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<true>(flags, database, table, column); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
-void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database) { revokeImpl<true>(flags, database); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<true>(flags, database, table); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<true>(flags, database, table, column); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
+void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
 void AccessRights::revokeGrantOption(const AccessRightsElement & element) { revokeImpl<true>(element); }
 void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { revokeImpl<true>(elements); }
 
@@ -984,20 +984,20 @@ bool AccessRights::isGrantedImpl(const AccessRightsElements & elements) const
 }
 
 bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl<false>(flags); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<false>(flags, database); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<false>(flags, database, table); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<false>(flags, database, table, column); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
-bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<false>(flags, database); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<false>(flags, database, table); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<false>(flags, database, table, column); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
+bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
 bool AccessRights::isGranted(const AccessRightsElement & element) const { return isGrantedImpl<false>(element); }
 bool AccessRights::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl<false>(elements); }
 
 bool AccessRights::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl<true>(flags); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<true>(flags, database); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<true>(flags, database, table); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<true>(flags, database, table, column); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
-bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<true>(flags, database); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<true>(flags, database, table); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<true>(flags, database, table, column); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
+bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
 bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl<true>(element); }
 bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl<true>(elements); }
 
@@ -1062,24 +1062,21 @@ void AccessRights::modifyFlags(const ModifyFlagsFunction & function)
 {
     if (!root)
         return;
+
     bool flags_added, flags_removed;
-    root->modifyFlags(function, flags_added, flags_removed);
+    root->modifyFlags(function, false, flags_added, flags_removed);
     if (flags_removed && root_with_grant_option)
         root_with_grant_option->makeIntersection(*root);
-}
 
-
-void AccessRights::modifyFlagsWithGrantOption(const ModifyFlagsFunction & function)
-{
-    if (!root_with_grant_option)
-        return;
-    bool flags_added, flags_removed;
-    root_with_grant_option->modifyFlags(function, flags_added, flags_removed);
-    if (flags_added)
+    if (root_with_grant_option)
     {
-        if (!root)
-            root = std::make_unique<Node>();
-        root->makeUnion(*root_with_grant_option);
+        root_with_grant_option->modifyFlags(function, true, flags_added, flags_removed);
+        if (flags_added)
+        {
+            if (!root)
+                root = std::make_unique<Node>();
+            root->makeUnion(*root_with_grant_option);
+        }
     }
 }
 

src/Access/AccessRights.h

@@ -39,59 +39,59 @@ public:
     /// Grants access on a specified database/table/column.
     /// Does nothing if the specified access has been already granted.
     void grant(const AccessFlags & flags);
-    void grant(const AccessFlags & flags, const std::string_view & database);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void grant(const AccessFlags & flags, std::string_view database);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void grant(const AccessRightsElement & element);
     void grant(const AccessRightsElements & elements);
 
     void grantWithGrantOption(const AccessFlags & flags);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void grantWithGrantOption(const AccessRightsElement & element);
     void grantWithGrantOption(const AccessRightsElements & elements);
 
     /// Revokes a specified access granted earlier on a specified database/table/column.
     /// For example, revoke(AccessType::ALL) revokes all grants at all, just like clear();
     void revoke(const AccessFlags & flags);
-    void revoke(const AccessFlags & flags, const std::string_view & database);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void revoke(const AccessFlags & flags, std::string_view database);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void revoke(const AccessRightsElement & element);
     void revoke(const AccessRightsElements & elements);
 
     void revokeGrantOption(const AccessFlags & flags);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
-    void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
+    void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
     void revokeGrantOption(const AccessRightsElement & element);
     void revokeGrantOption(const AccessRightsElements & elements);
 
     /// Whether a specified access granted.
     bool isGranted(const AccessFlags & flags) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool isGranted(const AccessRightsElement & element) const;
     bool isGranted(const AccessRightsElements & elements) const;
 
     bool hasGrantOption(const AccessFlags & flags) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool hasGrantOption(const AccessRightsElement & element) const;
     bool hasGrantOption(const AccessRightsElements & elements) const;
 
@@ -107,11 +107,11 @@ public:
         const AccessFlags & flags,
         const AccessFlags & min_flags_with_children,
         const AccessFlags & max_flags_with_children,
-        const std::string_view & database,
-        const std::string_view & table,
-        const std::string_view & column)>;
+        std::string_view database,
+        std::string_view table,
+        std::string_view column,
+        bool grant_option)>;
     void modifyFlags(const ModifyFlagsFunction & function);
-    void modifyFlagsWithGrantOption(const ModifyFlagsFunction & function);
 
     friend bool operator ==(const AccessRights & left, const AccessRights & right);
     friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); }

src/Access/Authentication.cpp

@@ -26,17 +26,17 @@ namespace
         return (Util::stringToDigest(password) == password_plaintext);
     }
 
-    bool checkPasswordDoubleSHA1(const std::string_view & password, const Digest & password_double_sha1)
+    bool checkPasswordDoubleSHA1(std::string_view password, const Digest & password_double_sha1)
     {
         return (Util::encodeDoubleSHA1(password) == password_double_sha1);
     }
 
-    bool checkPasswordSHA256(const std::string_view & password, const Digest & password_sha256, const String & salt)
+    bool checkPasswordSHA256(std::string_view password, const Digest & password_sha256, const String & salt)
     {
         return Util::encodeSHA256(String(password).append(salt)) == password_sha256;
     }
 
-    bool checkPasswordDoubleSHA1MySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_double_sha1)
+    bool checkPasswordDoubleSHA1MySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_double_sha1)
     {
         /// scrambled_password = SHA1(password) XOR SHA1(scramble <concat> SHA1(SHA1(password)))
 
@@ -61,7 +61,7 @@ namespace
         return calculated_password_double_sha1 == password_double_sha1;
     }
 
-    bool checkPasswordPlainTextMySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_plaintext)
+    bool checkPasswordPlainTextMySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_plaintext)
     {
         return checkPasswordDoubleSHA1MySQL(scramble, scrambled_password, Util::encodeDoubleSHA1(password_plaintext));
     }

src/Access/Common/AccessFlags.cpp

@@ -35,7 +35,7 @@ namespace
             return access_type_to_flags_mapping[static_cast<size_t>(type)];
         }
 
-        Flags keywordToFlags(const std::string_view & keyword) const
+        Flags keywordToFlags(std::string_view keyword) const
         {
             auto it = keyword_to_flags_map.find(keyword);
             if (it == keyword_to_flags_map.end())
@@ -142,14 +142,14 @@ namespace
             }
         };
 
-        static String replaceUnderscoreWithSpace(const std::string_view & str)
+        static String replaceUnderscoreWithSpace(std::string_view str)
         {
             String res{str};
             boost::replace_all(res, "_", " ");
             return res;
         }
 
-        static Strings splitAliases(const std::string_view & str)
+        static Strings splitAliases(std::string_view str)
         {
             Strings aliases;
             boost::split(aliases, str, boost::is_any_of(","));
@@ -160,10 +160,10 @@ namespace
 
         static void makeNode(
             AccessType access_type,
-            const std::string_view & name,
-            const std::string_view & aliases,
+            std::string_view name,
+            std::string_view aliases,
             NodeType node_type,
-            const std::string_view & parent_group_name,
+            std::string_view parent_group_name,
             std::unordered_map<std::string_view, Node *> & nodes,
             std::unordered_map<std::string_view, NodePtr> & owned_nodes,
             size_t & next_flag)
@@ -353,7 +353,7 @@ namespace
 
 
 AccessFlags::AccessFlags(AccessType type) : flags(Helper::instance().accessTypeToFlags(type)) {}
-AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
+AccessFlags::AccessFlags(std::string_view keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
 AccessFlags::AccessFlags(const std::vector<std::string_view> & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
 AccessFlags::AccessFlags(const Strings & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
 String AccessFlags::toString() const { return Helper::instance().flagsToString(flags); }

src/Access/Common/AccessFlags.h

@@ -21,7 +21,7 @@ public:
     AccessFlags() = default;
 
     /// Constructs from a string like "SELECT".
-    AccessFlags(const std::string_view & keyword); /// NOLINT
+    AccessFlags(std::string_view keyword); /// NOLINT
 
     /// Constructs from a list of strings like "SELECT, UPDATE, INSERT".
     AccessFlags(const std::vector<std::string_view> & keywords); /// NOLINT

src/Access/Common/AccessRightsElement.cpp

@@ -81,7 +81,7 @@ namespace
         }
 
         bool need_comma = false;
-        for (const std::string_view & keyword : keywords)
+        for (std::string_view keyword : keywords)
         {
             if (need_comma)
                 result.append(", ");
@@ -145,18 +145,18 @@ namespace
 }
 
 
-AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_)
+AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_)
     : access_flags(access_flags_), database(database_), any_database(false)
 {
 }
 
-AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_)
+AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_)
     : access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false)
 {
 }
 
 AccessRightsElement::AccessRightsElement(
-    AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_)
+    AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_)
     : access_flags(access_flags_)
     , database(database_)
     , table(table_)
@@ -169,8 +169,8 @@ AccessRightsElement::AccessRightsElement(
 
 AccessRightsElement::AccessRightsElement(
     AccessFlags access_flags_,
-    const std::string_view & database_,
-    const std::string_view & table_,
+    std::string_view database_,
+    std::string_view table_,
     const std::vector<std::string_view> & columns_)
     : access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false), any_column(false)
 {
@@ -180,7 +180,7 @@ AccessRightsElement::AccessRightsElement(
 }
 
 AccessRightsElement::AccessRightsElement(
-    AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_)
+    AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_)
     : access_flags(access_flags_)
     , database(database_)
     , table(table_)

src/Access/Common/AccessRightsElement.h

@@ -28,19 +28,19 @@ struct AccessRightsElement
 
     explicit AccessRightsElement(AccessFlags access_flags_) : access_flags(access_flags_) {}
 
-    AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_);
-    AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_);
+    AccessRightsElement(AccessFlags access_flags_, std::string_view database_);
+    AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_);
     AccessRightsElement(
-        AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_);
+        AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_);
 
     AccessRightsElement(
         AccessFlags access_flags_,
-        const std::string_view & database_,
-        const std::string_view & table_,
+        std::string_view database_,
+        std::string_view table_,
         const std::vector<std::string_view> & columns_);
 
     AccessRightsElement(
-        AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_);
+        AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_);
 
     bool empty() const { return !access_flags || (!any_column && columns.empty()); }
 

src/Access/Common/AccessType.cpp

@@ -35,7 +35,7 @@ namespace
 #undef ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING
         }
 
-        void addToMapping(AccessType type, const std::string_view & str)
+        void addToMapping(AccessType type, std::string_view str)
         {
             String str2{str};
             boost::replace_all(str2, "_", " ");

src/Access/Common/AuthenticationData.cpp

@@ -71,7 +71,7 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty
 }
 
 
-AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::string_view & text [[maybe_unused]])
+AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_view text [[maybe_unused]])
 {
 #if USE_SSL
     Digest hash;
@@ -86,7 +86,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::str
 }
 
 
-AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(const std::string_view & text)
+AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(std::string_view text)
 {
     Poco::SHA1Engine engine;
     engine.update(text.data(), text.size());

src/Access/Common/AuthenticationData.h

@@ -96,11 +96,11 @@ public:
 
     struct Util
     {
-        static Digest stringToDigest(const std::string_view & text) { return Digest(text.data(), text.data() + text.size()); }
-        static Digest encodeSHA256(const std::string_view & text);
-        static Digest encodeSHA1(const std::string_view & text);
+        static Digest stringToDigest(std::string_view text) { return Digest(text.data(), text.data() + text.size()); }
+        static Digest encodeSHA256(std::string_view text);
+        static Digest encodeSHA1(std::string_view text);
         static Digest encodeSHA1(const Digest & text) { return encodeSHA1(std::string_view{reinterpret_cast<const char *>(text.data()), text.size()}); }
-        static Digest encodeDoubleSHA1(const std::string_view & text) { return encodeSHA1(encodeSHA1(text)); }
+        static Digest encodeDoubleSHA1(std::string_view text) { return encodeSHA1(encodeSHA1(text)); }
         static Digest encodeDoubleSHA1(const Digest & text) { return encodeSHA1(encodeSHA1(text)); }
     };
 

src/Access/ContextAccess.cpp

@@ -44,9 +44,17 @@ namespace
     }
 
 
-    AccessRights addImplicitAccessRights(const AccessRights & access)
+    AccessRights addImplicitAccessRights(const AccessRights & access, const AccessControl & access_control)
     {
-        auto modifier = [&](const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, const std::string_view & database, const std::string_view & table, const std::string_view & column) -> AccessFlags
+        AccessFlags max_flags;
+
+        auto modifier = [&](const AccessFlags & flags,
+                            const AccessFlags & min_flags_with_children,
+                            const AccessFlags & max_flags_with_children,
+                            std::string_view database,
+                            std::string_view table,
+                            std::string_view column,
+                            bool /* grant_option */) -> AccessFlags
         {
             size_t level = !database.empty() + !table.empty() + !column.empty();
             AccessFlags res = flags;
@@ -115,17 +123,80 @@ namespace
                 res |= show_databases;
             }
 
+            max_flags |= res;
+
             return res;
         };
 
         AccessRights res = access;
         res.modifyFlags(modifier);
-        res.modifyFlagsWithGrantOption(modifier);
 
-        /// Anyone has access to the "system" and "information_schema" database.
-        res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
-        res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA);
-        res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE);
+        /// If "select_from_system_db_requires_grant" is enabled we provide implicit grants only for a few tables in the system database.
+        if (access_control.doesSelectFromSystemDatabaseRequireGrant())
+        {
+            const char * always_accessible_tables[] = {
+                /// Constant tables
+                "one",
+
+                /// "numbers", "numbers_mt", "zeros", "zeros_mt" were excluded because they can generate lots of values and
+                /// that can decrease performance in some cases.
+
+                "contributors",
+                "licenses",
+                "time_zones",
+                "collations",
+
+                "formats",
+                "privileges",
+                "data_type_families",
+                "table_engines",
+                "table_functions",
+                "aggregate_function_combinators",
+
+                "functions", /// Can contain user-defined functions
+
+                /// The following tables hide some rows if the current user doesn't have corresponding SHOW privileges.
+                "databases",
+                "tables",
+                "columns",
+
+                /// Specific to the current session
+                "settings",
+                "current_roles",
+                "enabled_roles",
+                "quota_usage"
+            };
+
+            for (const auto * table_name : always_accessible_tables)
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, table_name);
+
+            if (max_flags.contains(AccessType::SHOW_USERS))
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "users");
+
+            if (max_flags.contains(AccessType::SHOW_ROLES))
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "roles");
+
+            if (max_flags.contains(AccessType::SHOW_ROW_POLICIES))
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "row_policies");
+
+            if (max_flags.contains(AccessType::SHOW_SETTINGS_PROFILES))
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "settings_profiles");
+
+            if (max_flags.contains(AccessType::SHOW_QUOTAS))
+                res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "quotas");
+        }
+        else
+        {
+            res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
+        }
+
+        /// If "select_from_information_schema_requires_grant" is enabled we don't provide implicit grants for the information_schema database.
+        if (!access_control.doesSelectFromInformationSchemaRequireGrant())
+        {
+            res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA);
+            res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE);
+        }
+
         return res;
     }
 
@@ -141,7 +212,7 @@ namespace
     std::string_view getDatabase() { return {}; }
 
     template <typename... OtherArgs>
-    std::string_view getDatabase(const std::string_view & arg1, const OtherArgs &...) { return arg1; }
+    std::string_view getDatabase(std::string_view arg1, const OtherArgs &...) { return arg1; }
 }
 
 
@@ -247,7 +318,7 @@ void ContextAccess::setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> &
 void ContextAccess::calculateAccessRights() const
 {
     access = std::make_shared<AccessRights>(mixAccessRightsFromUserAndRoles(*user, *roles_info));
-    access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*access));
+    access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*access, *access_control));
 
     if (trace_log)
     {
@@ -342,7 +413,7 @@ std::shared_ptr<const ContextAccess> ContextAccess::getFullAccess()
         auto full_access = std::shared_ptr<ContextAccess>(new ContextAccess);
         full_access->is_full_access = true;
         full_access->access = std::make_shared<AccessRights>(AccessRights::getFullAccess());
-        full_access->access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*full_access->access));
+        full_access->access_with_implicit = full_access->access;
         return full_access;
     }();
     return res;
@@ -413,7 +484,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
     };
 
     if (is_full_access)
-        return access_granted();
+        return true;
 
     if (user_was_dropped)
         return access_denied("User has been dropped", ErrorCodes::UNKNOWN_USER);
@@ -422,7 +493,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
         flags &= ~AccessType::CLUSTER;
 
     if (!flags)
-        return access_granted();
+        return true;
 
     /// Access to temporary tables is controlled in an unusual way, not like normal tables.
     /// Creating of temporary tables is controlled by AccessType::CREATE_TEMPORARY_TABLES grant,
@@ -519,7 +590,7 @@ bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const
 }
 
 template <bool throw_if_denied, bool grant_option, typename... Args>
-bool ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const
+bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const
 {
     return checkAccessImplHelper<throw_if_denied, grant_option>(flags, database.empty() ? params.current_database : database, args...);
 }
@@ -564,38 +635,38 @@ bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
 }
 
 bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl<false, false>(flags); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, false>(flags, database); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, false>(flags, database, table); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
-bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(flags, database); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(flags, database, table); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
+bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
 bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl<false, false>(element); }
 bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(elements); }
 
 bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return checkAccessImpl<false, true>(flags); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, true>(flags, database); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, true>(flags, database, table); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
-bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(flags, database); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(flags, database, table); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
+bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
 bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return checkAccessImpl<false, true>(element); }
 bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(elements); }
 
 void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<true, false>(flags); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, false>(flags, database); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, false>(flags, database, table); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, false>(flags, database, table, column); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
-void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(flags, database); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(flags, database, table); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(flags, database, table, column); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
+void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
 void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<true, false>(element); }
 void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<true, false>(elements); }
 
 void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<true, true>(flags); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, true>(flags, database); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, true>(flags, database, table); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, true>(flags, database, table, column); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
-void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(flags, database); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(flags, database, table); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(flags, database, table, column); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
+void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
 void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<true, true>(element); }
 void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true, true>(elements); }
 

src/Access/ContextAccess.h

@@ -101,40 +101,40 @@ public:
     /// Checks if a specified access is granted, and throws an exception if not.
     /// Empty database means the current database.
     void checkAccess(const AccessFlags & flags) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     void checkAccess(const AccessRightsElement & element) const;
     void checkAccess(const AccessRightsElements & elements) const;
 
     void checkGrantOption(const AccessFlags & flags) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     void checkGrantOption(const AccessRightsElement & element) const;
     void checkGrantOption(const AccessRightsElements & elements) const;
 
     /// Checks if a specified access is granted, and returns false if not.
     /// Empty database means the current database.
     bool isGranted(const AccessFlags & flags) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool isGranted(const AccessRightsElement & element) const;
     bool isGranted(const AccessRightsElements & elements) const;
 
     bool hasGrantOption(const AccessFlags & flags) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
-    bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
+    bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
     bool hasGrantOption(const AccessRightsElement & element) const;
     bool hasGrantOption(const AccessRightsElements & elements) const;
 
@@ -180,7 +180,7 @@ private:
     bool checkAccessImpl(const AccessFlags & flags) const;
 
     template <bool throw_if_denied, bool grant_option, typename... Args>
-    bool checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const;
+    bool checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const;
 
     template <bool throw_if_denied, bool grant_option>
     bool checkAccessImpl(const AccessRightsElement & element) const;

src/Access/SettingsConstraints.cpp

@@ -36,12 +36,12 @@ void SettingsConstraints::clear()
 }
 
 
-void SettingsConstraints::setMinValue(const std::string_view & setting_name, const Field & min_value)
+void SettingsConstraints::setMinValue(std::string_view setting_name, const Field & min_value)
 {
     getConstraintRef(setting_name).min_value = Settings::castValueUtil(setting_name, min_value);
 }
 
-Field SettingsConstraints::getMinValue(const std::string_view & setting_name) const
+Field SettingsConstraints::getMinValue(std::string_view setting_name) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -51,12 +51,12 @@ Field SettingsConstraints::getMinValue(const std::string_view & setting_name) co
 }
 
 
-void SettingsConstraints::setMaxValue(const std::string_view & setting_name, const Field & max_value)
+void SettingsConstraints::setMaxValue(std::string_view setting_name, const Field & max_value)
 {
     getConstraintRef(setting_name).max_value = Settings::castValueUtil(setting_name, max_value);
 }
 
-Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) const
+Field SettingsConstraints::getMaxValue(std::string_view setting_name) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -66,12 +66,12 @@ Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) co
 }
 
 
-void SettingsConstraints::setReadOnly(const std::string_view & setting_name, bool read_only)
+void SettingsConstraints::setReadOnly(std::string_view setting_name, bool read_only)
 {
     getConstraintRef(setting_name).read_only = read_only;
 }
 
-bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) const
+bool SettingsConstraints::isReadOnly(std::string_view setting_name) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -81,7 +81,7 @@ bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) cons
 }
 
 
-void SettingsConstraints::set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only)
+void SettingsConstraints::set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only)
 {
     auto & ref = getConstraintRef(setting_name);
     ref.min_value = Settings::castValueUtil(setting_name, min_value);
@@ -89,7 +89,7 @@ void SettingsConstraints::set(const std::string_view & setting_name, const Field
     ref.read_only = read_only;
 }
 
-void SettingsConstraints::get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const
+void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const
 {
     const auto * ptr = tryGetConstraint(setting_name);
     if (ptr)
@@ -318,7 +318,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh
 }
 
 
-SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const std::string_view & setting_name)
+SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(std::string_view setting_name)
 {
     auto it = constraints.find(setting_name);
     if (it == constraints.end())
@@ -331,7 +331,7 @@ SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const st
     return it->second;
 }
 
-const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(const std::string_view & setting_name) const
+const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(std::string_view setting_name) const
 {
     auto it = constraints.find(setting_name);
     if (it == constraints.end())

src/Access/SettingsConstraints.h

@@ -61,17 +61,17 @@ public:
     void clear();
     bool empty() const { return constraints.empty(); }
 
-    void setMinValue(const std::string_view & setting_name, const Field & min_value);
-    Field getMinValue(const std::string_view & setting_name) const;
+    void setMinValue(std::string_view setting_name, const Field & min_value);
+    Field getMinValue(std::string_view setting_name) const;
 
-    void setMaxValue(const std::string_view & setting_name, const Field & max_value);
-    Field getMaxValue(const std::string_view & setting_name) const;
+    void setMaxValue(std::string_view setting_name, const Field & max_value);
+    Field getMaxValue(std::string_view setting_name) const;
 
-    void setReadOnly(const std::string_view & setting_name, bool read_only);
-    bool isReadOnly(const std::string_view & setting_name) const;
+    void setReadOnly(std::string_view setting_name, bool read_only);
+    bool isReadOnly(std::string_view setting_name) const;
 
-    void set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only);
-    void get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const;
+    void set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only);
+    void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const;
 
     void merge(const SettingsConstraints & other);
 
@@ -105,8 +105,8 @@ private:
     };
     bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const;
 
-    Constraint & getConstraintRef(const std::string_view & setting_name);
-    const Constraint * tryGetConstraint(const std::string_view & setting_name) const;
+    Constraint & getConstraintRef(std::string_view setting_name);
+    const Constraint * tryGetConstraint(std::string_view setting_name) const;
 
     std::unordered_map<std::string_view, Constraint> constraints;
     const AccessControl * access_control = nullptr;

src/AggregateFunctions/ThetaSketchData.h

@@ -43,7 +43,7 @@ public:
     ~ThetaSketchData() = default;
 
     /// Insert original value without hash, as `datasketches::update_theta_sketch.update` will do the hash internal.
-    void insertOriginal(const StringRef & value)
+    void insertOriginal(StringRef value)
     {
         getSkUpdate()->update(value.data, value.size);
     }

src/Backups/BackupsWorker.cpp

@@ -107,7 +107,7 @@ UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & c
         {
             if (async)
             {
-                query_scope.emplace(context_in_use);
+                query_scope.emplace(mutable_context);
                 setThreadName("BackupWorker");
             }
 

src/Client/ClientBase.cpp

@@ -601,6 +601,7 @@ void ClientBase::initLogsOutputStream()
     {
         WriteBuffer * wb = out_logs_buf.get();
 
+        bool color_logs = false;
         if (!out_logs_buf)
         {
             if (server_logs_file.empty())
@@ -608,11 +609,13 @@ void ClientBase::initLogsOutputStream()
                 /// Use stderr by default
                 out_logs_buf = std::make_unique<WriteBufferFromFileDescriptor>(STDERR_FILENO);
                 wb = out_logs_buf.get();
+                color_logs = stderr_is_a_tty;
             }
             else if (server_logs_file == "-")
             {
                 /// Use stdout if --server_logs_file=- specified
                 wb = &std_out;
+                color_logs = stdout_is_a_tty;
             }
             else
             {
@@ -622,7 +625,7 @@ void ClientBase::initLogsOutputStream()
             }
         }
 
-        logs_out_stream = std::make_unique<InternalTextLogs>(*wb, stdout_is_a_tty);
+        logs_out_stream = std::make_unique<InternalTextLogs>(*wb, color_logs);
     }
 }
 
@@ -2155,6 +2158,7 @@ void ClientBase::init(int argc, char ** argv)
 
     stdin_is_a_tty = isatty(STDIN_FILENO);
     stdout_is_a_tty = isatty(STDOUT_FILENO);
+    stderr_is_a_tty = isatty(STDERR_FILENO);
     terminal_width = getTerminalWidth();
 
     Arguments common_arguments{""}; /// 0th argument is ignored.

src/Client/ClientBase.h

@@ -173,6 +173,7 @@ protected:
 
     bool stdin_is_a_tty = false; /// stdin is a terminal.
     bool stdout_is_a_tty = false; /// stdout is a terminal.
+    bool stderr_is_a_tty = false; /// stderr is a terminal.
     uint64_t terminal_width = 0;
 
     ServerConnectionPtr connection;

src/Client/Suggest.cpp

@@ -50,52 +50,58 @@ static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggesti
 {
     /// NOTE: Once you will update the completion list,
     /// do not forget to update 01676_clickhouse_client_autocomplete.sh
-    WriteBufferFromOwnString query;
-    query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM ("
-        "SELECT name FROM system.functions"
-        " UNION ALL "
-        "SELECT name FROM system.table_engines"
-        " UNION ALL "
-        "SELECT name FROM system.formats"
-        " UNION ALL "
-        "SELECT name FROM system.table_functions"
-        " UNION ALL "
-        "SELECT name FROM system.data_type_families"
-        " UNION ALL "
-        "SELECT name FROM system.merge_tree_settings"
-        " UNION ALL "
-        "SELECT name FROM system.settings"
-        " UNION ALL ";
+    String query;
+
+    auto add_subquery = [&](std::string_view select, std::string_view result_column_name)
+    {
+        if (!query.empty())
+            query += " UNION ALL ";
+        query += fmt::format("SELECT * FROM viewIfPermitted({} ELSE null('{} String'))", select, result_column_name);
+    };
+
+    auto add_column = [&](std::string_view column_name, std::string_view table_name, bool distinct, std::optional<Int64> limit)
+    {
+        add_subquery(
+            fmt::format(
+                "SELECT {}{} FROM system.{}{}",
+                (distinct ? "DISTINCT " : ""),
+                column_name,
+                table_name,
+                (limit ? (" LIMIT " + std::to_string(*limit)) : "")),
+            column_name);
+    };
+
+    add_column("name", "functions", false, {});
+    add_column("name", "table_engines", false, {});
+    add_column("name", "formats", false, {});
+    add_column("name", "table_functions", false, {});
+    add_column("name", "data_type_families", false, {});
+    add_column("name", "merge_tree_settings", false, {});
+    add_column("name", "settings", false, {});
+
     if (!basic_suggestion)
     {
-        query << "SELECT cluster FROM system.clusters"
-                 " UNION ALL "
-                 "SELECT macro FROM system.macros"
-                 " UNION ALL "
-                 "SELECT policy_name FROM system.storage_policies"
-                 " UNION ALL ";
+        add_column("cluster", "clusters", false, {});
+        add_column("macro", "macros", false, {});
+        add_column("policy_name", "storage_policies", false, {});
     }
-    query << "SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate";
+
+    add_subquery("SELECT concat(func.name, comb.name) AS x FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate", "x");
+
     /// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero.
     if (suggestion_limit > 0)
     {
-        String limit_str = toString(suggestion_limit);
-        query << " UNION ALL "
-                 "SELECT name FROM system.databases LIMIT " << limit_str
-              << " UNION ALL "
-                 "SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
-              << " UNION ALL ";
-
+        add_column("name", "databases", false, suggestion_limit);
+        add_column("name", "tables", true, suggestion_limit);
         if (!basic_suggestion)
         {
-            query << "SELECT DISTINCT name FROM system.dictionaries LIMIT " << limit_str
-                  << " UNION ALL ";
+            add_column("name", "dictionaries", true, suggestion_limit);
         }
-        query << "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
+        add_column("name", "columns", true, suggestion_limit);
     }
-    query << ") WHERE notEmpty(res)";
 
-    return query.str();
+    query = "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + query + ") WHERE notEmpty(res)";
+    return query;
 }
 
 template <typename ConnectionType>

src/Columns/ColumnUnique.h

@@ -509,7 +509,7 @@ MutableColumnPtr ColumnUnique<ColumnType>::uniqueInsertRangeImpl(
     if (secondary_index)
         next_position += secondary_index->size();
 
-    auto insert_key = [&](const StringRef & ref, ReverseIndex<UInt64, ColumnType> & cur_index) -> MutableColumnPtr
+    auto insert_key = [&](StringRef ref, ReverseIndex<UInt64, ColumnType> & cur_index) -> MutableColumnPtr
     {
         auto inserted_pos = cur_index.insert(ref);
         positions[num_added_rows] = inserted_pos;

src/Columns/ReverseIndex.h

@@ -92,7 +92,7 @@ struct ReverseIndexHashTableCell
 
     /// Special case when we want to compare with something not in index_column.
     /// When we compare something inside column default keyEquals checks only that row numbers are equal.
-    bool keyEquals(const StringRef & object, size_t hash_ [[maybe_unused]], const State & state) const
+    bool keyEquals(StringRef object, size_t hash_ [[maybe_unused]], const State & state) const
     {
         auto index = key;
         if constexpr (has_base_index)
@@ -322,7 +322,7 @@ public:
     static constexpr bool is_numeric_column = isNumericColumn(static_cast<ColumnType *>(nullptr));
     static constexpr bool use_saved_hash = !is_numeric_column;
 
-    UInt64 insert(const StringRef & data);
+    UInt64 insert(StringRef data);
 
     /// Returns the found data's index in the dictionary. If index is not built, builds it.
     UInt64 getInsertionPoint(StringRef data)
@@ -383,7 +383,7 @@ private:
 
     void buildIndex();
 
-    UInt64 getHash(const StringRef & ref) const
+    UInt64 getHash(StringRef ref) const
     {
         if constexpr (is_numeric_column)
         {
@@ -478,7 +478,7 @@ ColumnUInt64::MutablePtr ReverseIndex<IndexType, ColumnType>::calcHashes() const
 }
 
 template <typename IndexType, typename ColumnType>
-UInt64 ReverseIndex<IndexType, ColumnType>::insert(const StringRef & data)
+UInt64 ReverseIndex<IndexType, ColumnType>::insert(StringRef data)
 {
     if (!index)
         buildIndex();

src/Common/HashTable/StringHashMap.h

@@ -12,7 +12,7 @@ struct StringHashMapCell : public HashMapCell<Key, TMapped, StringHashTableHash,
     using Base::Base;
     static constexpr bool need_zero_value_storage = false;
     // external
-    const StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
+    StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
     // internal
     static const Key & getKey(const value_type & value_) { return value_.first; }
 };
@@ -32,7 +32,7 @@ struct StringHashMapCell<StringKey16, TMapped> : public HashMapCell<StringKey16,
     void setZero() { this->value.first.items[1] = 0; }
 
     // external
-    const StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
+    StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
     // internal
     static const StringKey16 & getKey(const value_type & value_) { return value_.first; }
 };
@@ -53,7 +53,7 @@ struct StringHashMapCell<StringKey24, TMapped> : public HashMapCell<StringKey24,
     void setZero() { this->value.first.c = 0; }
 
     // external
-    const StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
+    StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
     // internal
     static const StringKey24 & getKey(const value_type & value_) { return value_.first; }
 };

src/Common/JSONParsers/DummyJSONParser.h

@@ -84,7 +84,7 @@ struct DummyJSONParser
         static Iterator begin() { return {}; }
         static Iterator end() { return {}; }
         static size_t size() { return 0; }
-        bool find(const std::string_view &, Element &) const { return false; } /// NOLINT
+        bool find(std::string_view, Element &) const { return false; } /// NOLINT
 
 #if 0
         /// Optional: Provides access to an object's element by index.
@@ -93,7 +93,7 @@ struct DummyJSONParser
     };
 
     /// Parses a JSON document, returns the reference to its root element if succeeded.
-    bool parse(const std::string_view &, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
+    bool parse(std::string_view, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
 
 #if 0
     /// Optional: Allocates memory to parse JSON documents faster.

src/Common/JSONParsers/RapidJSONParser.h

@@ -98,7 +98,7 @@ struct RapidJSONParser
         ALWAYS_INLINE Iterator end() const { return ptr->MemberEnd(); }
         ALWAYS_INLINE size_t size() const { return ptr->MemberCount(); }
 
-        bool find(const std::string_view & key, Element & result) const
+        bool find(std::string_view key, Element & result) const
         {
             auto it = ptr->FindMember(rapidjson::StringRef(key.data(), key.length()));
             if (it == ptr->MemberEnd())
@@ -122,7 +122,7 @@ struct RapidJSONParser
     };
 
     /// Parses a JSON document, returns the reference to its root element if succeeded.
-    bool parse(const std::string_view & json, Element & result)
+    bool parse(std::string_view json, Element & result)
     {
         rapidjson::MemoryStream ms(json.data(), json.size());
         rapidjson::EncodedInputStream<rapidjson::UTF8<>, rapidjson::MemoryStream> is(ms);

src/Common/JSONParsers/SimdJSONParser.h

@@ -105,7 +105,7 @@ struct SimdJSONParser
         ALWAYS_INLINE Iterator end() const { return object.end(); }
         ALWAYS_INLINE size_t size() const { return object.size(); }
 
-        bool find(const std::string_view & key, Element & result) const
+        bool find(std::string_view key, Element & result) const
         {
             auto x = object.at_key(key);
             if (x.error())
@@ -131,7 +131,7 @@ struct SimdJSONParser
     };
 
     /// Parses a JSON document, returns the reference to its root element if succeeded.
-    bool parse(const std::string_view & json, Element & result)
+    bool parse(std::string_view json, Element & result)
     {
         auto document = parser.parse(json.data(), json.size());
         if (document.error())

src/Common/OpenSSLHelpers.cpp

@@ -10,7 +10,7 @@ namespace DB
 {
 #pragma GCC diagnostic warning "-Wold-style-cast"
 
-std::string encodeSHA256(const std::string_view & text)
+std::string encodeSHA256(std::string_view text)
 {
     return encodeSHA256(text.data(), text.size());
 }
@@ -21,7 +21,7 @@ std::string encodeSHA256(const void * text, size_t size)
     encodeSHA256(text, size, reinterpret_cast<unsigned char *>(out.data()));
     return out;
 }
-void encodeSHA256(const std::string_view & text, unsigned char * out)
+void encodeSHA256(std::string_view text, unsigned char * out)
 {
     encodeSHA256(text.data(), text.size(), out);
 }

src/Common/OpenSSLHelpers.h

@@ -10,10 +10,10 @@ namespace DB
 {
 
 /// Encodes `text` and returns it.
-std::string encodeSHA256(const std::string_view & text);
+std::string encodeSHA256(std::string_view text);
 std::string encodeSHA256(const void * text, size_t size);
 /// `out` must be at least 32 bytes long.
-void encodeSHA256(const std::string_view & text, unsigned char * out);
+void encodeSHA256(std::string_view text, unsigned char * out);
 void encodeSHA256(const void * text, size_t size, unsigned char * out);
 
 /// Returns concatenation of error strings for all errors that OpenSSL has recorded, emptying the error queue.

src/Common/SettingsChanges.cpp

@@ -4,7 +4,7 @@ namespace DB
 {
 namespace
 {
-    SettingChange * find(SettingsChanges & changes, const std::string_view & name)
+    SettingChange * find(SettingsChanges & changes, std::string_view name)
     {
         auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
         if (it == changes.end())
@@ -12,7 +12,7 @@ namespace
         return &*it;
     }
 
-    const SettingChange * find(const SettingsChanges & changes, const std::string_view & name)
+    const SettingChange * find(const SettingsChanges & changes, std::string_view name)
     {
         auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
         if (it == changes.end())
@@ -21,7 +21,7 @@ namespace
     }
 }
 
-bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) const
+bool SettingsChanges::tryGet(std::string_view name, Field & out_value) const
 {
     const auto * change = find(*this, name);
     if (!change)
@@ -30,7 +30,7 @@ bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) c
     return true;
 }
 
-const Field * SettingsChanges::tryGet(const std::string_view & name) const
+const Field * SettingsChanges::tryGet(std::string_view name) const
 {
     const auto * change = find(*this, name);
     if (!change)
@@ -38,7 +38,7 @@ const Field * SettingsChanges::tryGet(const std::string_view & name) const
     return &change->value;
 }
 
-Field * SettingsChanges::tryGet(const std::string_view & name)
+Field * SettingsChanges::tryGet(std::string_view name)
 {
     auto * change = find(*this, name);
     if (!change)

src/Common/SettingsChanges.h

@@ -14,8 +14,8 @@ struct SettingChange
     Field value;
 
     SettingChange() = default;
-    SettingChange(const std::string_view & name_, const Field & value_) : name(name_), value(value_) {}
-    SettingChange(const std::string_view & name_, Field && value_) : name(name_), value(std::move(value_)) {}
+    SettingChange(std::string_view name_, const Field & value_) : name(name_), value(value_) {}
+    SettingChange(std::string_view name_, Field && value_) : name(name_), value(std::move(value_)) {}
 
     friend bool operator ==(const SettingChange & lhs, const SettingChange & rhs) { return (lhs.name == rhs.name) && (lhs.value == rhs.value); }
     friend bool operator !=(const SettingChange & lhs, const SettingChange & rhs) { return !(lhs == rhs); }
@@ -27,9 +27,9 @@ class SettingsChanges : public std::vector<SettingChange>
 public:
     using std::vector<SettingChange>::vector;
 
-    bool tryGet(const std::string_view & name, Field & out_value) const;
-    const Field * tryGet(const std::string_view & name) const;
-    Field * tryGet(const std::string_view & name);
+    bool tryGet(std::string_view name, Field & out_value) const;
+    const Field * tryGet(std::string_view name) const;
+    Field * tryGet(std::string_view name);
 };
 
 }

src/Common/SpaceSaving.h

@@ -49,12 +49,12 @@ struct SpaceSavingArena
 template <>
 struct SpaceSavingArena<StringRef>
 {
-    StringRef emplace(const StringRef & key)
+    StringRef emplace(StringRef key)
     {
         return copyStringInArena(arena, key);
     }
 
-    void free(const StringRef & key)
+    void free(StringRef key)
     {
         if (key.data)
             arena.free(const_cast<char *>(key.data), key.size);

src/Common/StringUtils/StringUtils.h

@@ -147,7 +147,7 @@ inline bool isPunctuationASCII(char c)
 }
 
 
-inline bool isValidIdentifier(const std::string_view & str)
+inline bool isValidIdentifier(std::string_view str)
 {
     return !str.empty()
         && isValidIdentifierBegin(str[0])

src/Common/TLDListsHolder.cpp

@@ -20,13 +20,13 @@ TLDList::TLDList(size_t size)
     : tld_container(size)
     , pool(std::make_unique<Arena>(10 << 20))
 {}
-bool TLDList::insert(const StringRef & host)
+bool TLDList::insert(StringRef host)
 {
     bool inserted;
     tld_container.emplace(DB::ArenaKeyHolder{host, *pool}, inserted);
     return inserted;
 }
-bool TLDList::has(const StringRef & host) const
+bool TLDList::has(StringRef host) const
 {
     return tld_container.has(host);
 }

src/Common/TLDListsHolder.h

@@ -23,9 +23,9 @@ public:
     explicit TLDList(size_t size);
 
     /// Return true if the tld_container does not contains such element.
-    bool insert(const StringRef & host);
+    bool insert(StringRef host);
     /// Check is there such TLD
-    bool has(const StringRef & host) const;
+    bool has(StringRef host) const;
     size_t size() const { return tld_container.size(); }
 
 private:

src/Common/TaskStatsInfoGetter.cpp

@@ -21,6 +21,7 @@
 
 #if defined(__clang__)
     #pragma clang diagnostic ignored "-Wgnu-anonymous-struct"
+    #pragma clang diagnostic ignored "-Wnested-anon-types"
 #endif
 
 /// Basic idea is motivated by "iotop" tool.

src/Common/Throttler.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <Common/Throttler_fwd.h>
+
 #include <mutex>
 #include <memory>
 #include <base/sleep.h>
@@ -57,7 +59,4 @@ private:
     std::shared_ptr<Throttler> parent;
 };
 
-
-using ThrottlerPtr = std::shared_ptr<Throttler>;
-
 }

src/Common/Throttler_fwd.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <memory>
+
+namespace DB
+{
+
+class Throttler;
+using ThrottlerPtr = std::shared_ptr<Throttler>;
+
+}

src/Common/filesystemHelpers.cpp

@@ -87,7 +87,10 @@ BlockDeviceType getBlockDeviceType([[maybe_unused]] const String & device_id)
 #if defined(OS_LINUX)
     try
     {
-        ReadBufferFromFile in("/sys/dev/block/" + device_id + "/queue/rotational");
+        const auto path{std::filesystem::path("/sys/dev/block/") / device_id / "queue/rotational"};
+        if (!std::filesystem::exists(path))
+            return BlockDeviceType::UNKNOWN;
+        ReadBufferFromFile in(path);
         int rotational;
         readText(rotational, in);
         return rotational ? BlockDeviceType::ROT : BlockDeviceType::NONROT;
@@ -109,7 +112,8 @@ UInt64 getBlockDeviceReadAheadBytes([[maybe_unused]] const String & device_id)
 #if defined(OS_LINUX)
     try
     {
-        ReadBufferFromFile in("/sys/dev/block/" + device_id + "/queue/read_ahead_kb");
+        const auto path{std::filesystem::path("/sys/dev/block/") / device_id / "queue/read_ahead_kb"};
+        ReadBufferFromFile in(path);
         int read_ahead_kb;
         readText(read_ahead_kb, in);
         return read_ahead_kb * 1024;

src/Common/noexcept_scope.h

@@ -1,36 +1,28 @@
 #pragma once
-#include <base/scope_guard.h>
 #include <Common/Exception.h>
 #include <Common/LockMemoryExceptionInThread.h>
 
-
-#define NOEXCEPT_SCOPE_IMPL_CONCAT(n, expected) \
-    LockMemoryExceptionInThread lock_memory_tracker##n(VariableContext::Global);   \
-    SCOPE_EXIT(                                                                    \
-        {                                                                          \
-            const auto uncaught = std::uncaught_exceptions();                      \
-            assert((expected) == uncaught || (expected) + 1 == uncaught);          \
-            if ((expected) < uncaught)                                             \
-            {                                                                      \
-                tryLogCurrentException("NOEXCEPT_SCOPE");                          \
-                abort();                                                           \
-            }                                                                      \
-        }                                                                          \
-    )
-
-#define NOEXCEPT_SCOPE_IMPL(n, expected) NOEXCEPT_SCOPE_IMPL_CONCAT(n, expected)
-
-#define NOEXCEPT_SCOPE_CONCAT(n)                                                   \
-    const auto num_curr_exceptions##n = std::uncaught_exceptions();                \
-    NOEXCEPT_SCOPE_IMPL(n, num_curr_exceptions##n)
-
-#define NOEXCEPT_SCOPE_FWD(n) NOEXCEPT_SCOPE_CONCAT(n)
-
-
 /// It can be used in critical places to exit on unexpected exceptions.
 /// SIGABRT is usually better that broken in-memory state with unpredictable consequences.
 /// It also temporarily disables exception from memory tracker in current thread.
 /// Strict version does not take into account nested exception (i.e. it aborts even when we're in catch block).
 
-#define NOEXCEPT_SCOPE_STRICT NOEXCEPT_SCOPE_IMPL(__LINE__, 0)
-#define NOEXCEPT_SCOPE NOEXCEPT_SCOPE_FWD(__LINE__)
+#define NOEXCEPT_SCOPE_IMPL(...) do {                          \
+    LockMemoryExceptionInThread                                \
+        noexcept_lock_memory_tracker(VariableContext::Global); \
+    try                                                        \
+    {                                                          \
+        __VA_ARGS__;                                           \
+    }                                                          \
+    catch (...)                                                \
+    {                                                          \
+        DB::tryLogCurrentException(__PRETTY_FUNCTION__);       \
+        std::terminate();                                      \
+    }                                                          \
+} while (0) /* to allow leading semi-colon */
+
+#define NOEXCEPT_SCOPE_STRICT(...)                    \
+    if (std::uncaught_exceptions()) std::terminate(); \
+    NOEXCEPT_SCOPE_IMPL(__VA_ARGS__)
+
+#define NOEXCEPT_SCOPE(...) NOEXCEPT_SCOPE_IMPL(__VA_ARGS__)

src/Common/quoteString.cpp

@@ -14,7 +14,7 @@ String quoteString(std::string_view x)
 }
 
 
-String doubleQuoteString(const StringRef & x)
+String doubleQuoteString(StringRef x)
 {
     String res(x.size, '\0');
     WriteBufferFromString wb(res);
@@ -23,7 +23,7 @@ String doubleQuoteString(const StringRef & x)
 }
 
 
-String backQuote(const StringRef & x)
+String backQuote(StringRef x)
 {
     String res(x.size, '\0');
     {
@@ -34,7 +34,7 @@ String backQuote(const StringRef & x)
 }
 
 
-String backQuoteIfNeed(const StringRef & x)
+String backQuoteIfNeed(StringRef x)
 {
     String res(x.size, '\0');
     {

src/Common/quoteString.h

@@ -16,12 +16,12 @@ namespace DB
 }
 
 /// Double quote the string.
-String doubleQuoteString(const StringRef & x);
+String doubleQuoteString(StringRef x);
 
 /// Quote the identifier with backquotes.
-String backQuote(const StringRef & x);
+String backQuote(StringRef x);
 
 /// Quote the identifier with backquotes, if required.
-String backQuoteIfNeed(const StringRef & x);
+String backQuoteIfNeed(StringRef x);
 
 }

src/Compression/CompressionCodecEncrypted.cpp

@@ -131,7 +131,7 @@ std::string lastErrorString()
 /// This function get key and nonce and encrypt text with their help.
 /// If something went wrong (can't init context or can't encrypt data) it throws exception.
 /// It returns length of encrypted text.
-size_t encrypt(const std::string_view & plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce)
+size_t encrypt(std::string_view plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce)
 {
     /// Init context for encryption, using key.
     EVP_AEAD_CTX encrypt_ctx;
@@ -160,7 +160,7 @@ size_t encrypt(const std::string_view & plaintext, char * ciphertext_and_tag, En
 /// This function get key and nonce and encrypt text with their help.
 /// If something went wrong (can't init context or can't encrypt data) it throws exception.
 /// It returns length of encrypted text.
-size_t decrypt(const std::string_view & ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce)
+size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce)
 {
     /// Init context for decryption with given key.
     EVP_AEAD_CTX decrypt_ctx;

src/Core/BaseSettings.cpp

@@ -11,7 +11,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_SETTING;
 }
 
-void BaseSettingsHelpers::writeString(const std::string_view & str, WriteBuffer & out)
+void BaseSettingsHelpers::writeString(std::string_view str, WriteBuffer & out)
 {
     writeStringBinary(str, out);
 }
@@ -39,13 +39,13 @@ BaseSettingsHelpers::Flags BaseSettingsHelpers::readFlags(ReadBuffer & in)
 }
 
 
-void BaseSettingsHelpers::throwSettingNotFound(const std::string_view & name)
+void BaseSettingsHelpers::throwSettingNotFound(std::string_view name)
 {
     throw Exception("Unknown setting " + String{name}, ErrorCodes::UNKNOWN_SETTING);
 }
 
 
-void BaseSettingsHelpers::warningSettingNotFound(const std::string_view & name)
+void BaseSettingsHelpers::warningSettingNotFound(std::string_view name)
 {
     static auto * log = &Poco::Logger::get("Settings");
     LOG_WARNING(log, "Unknown setting {}, skipping", name);

src/Core/BaseSettings.h

@@ -45,16 +45,16 @@ class BaseSettings : public TTraits::Data
 public:
     using Traits = TTraits;
 
-    void set(const std::string_view & name, const Field & value);
-    Field get(const std::string_view & name) const;
+    void set(std::string_view name, const Field & value);
+    Field get(std::string_view name) const;
 
-    void setString(const std::string_view & name, const String & value);
-    String getString(const std::string_view & name) const;
+    void setString(std::string_view name, const String & value);
+    String getString(std::string_view name) const;
 
-    bool tryGet(const std::string_view & name, Field & value) const;
-    bool tryGetString(const std::string_view & name, String & value) const;
+    bool tryGet(std::string_view name, Field & value) const;
+    bool tryGetString(std::string_view name, String & value) const;
 
-    bool isChanged(const std::string_view & name) const;
+    bool isChanged(std::string_view name) const;
     SettingsChanges changes() const;
     void applyChange(const SettingChange & change);
     void applyChanges(const SettingsChanges & changes);
@@ -63,22 +63,22 @@ public:
     /// Resets all the settings to their default values.
     void resetToDefault();
 
-    bool has(const std::string_view & name) const { return hasBuiltin(name) || hasCustom(name); }
-    static bool hasBuiltin(const std::string_view & name);
-    bool hasCustom(const std::string_view & name) const;
+    bool has(std::string_view name) const { return hasBuiltin(name) || hasCustom(name); }
+    static bool hasBuiltin(std::string_view name);
+    bool hasCustom(std::string_view name) const;
 
-    const char * getTypeName(const std::string_view & name) const;
-    const char * getDescription(const std::string_view & name) const;
+    const char * getTypeName(std::string_view name) const;
+    const char * getDescription(std::string_view name) const;
 
     /// Checks if it's possible to assign a field to a specified value and throws an exception if not.
     /// This function doesn't change the fields, it performs check only.
-    static void checkCanSet(const std::string_view & name, const Field & value);
-    static void checkCanSetString(const std::string_view & name, const String & str);
+    static void checkCanSet(std::string_view name, const Field & value);
+    static void checkCanSetString(std::string_view name, const String & str);
 
     /// Conversions without changing the fields.
-    static Field castValueUtil(const std::string_view & name, const Field & value);
-    static String valueToStringUtil(const std::string_view & name, const Field & value);
-    static Field stringToValueUtil(const std::string_view & name, const String & str);
+    static Field castValueUtil(std::string_view name, const Field & value);
+    static String valueToStringUtil(std::string_view name, const Field & value);
+    static Field stringToValueUtil(std::string_view name, const String & str);
 
     void write(WriteBuffer & out, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT) const;
     void read(ReadBuffer & in, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT);
@@ -164,19 +164,19 @@ public:
     Iterator end() const { return allChanged().end(); }
 
 private:
-    SettingFieldCustom & getCustomSetting(const std::string_view & name);
-    const SettingFieldCustom & getCustomSetting(const std::string_view & name) const;
-    const SettingFieldCustom * tryGetCustomSetting(const std::string_view & name) const;
+    SettingFieldCustom & getCustomSetting(std::string_view name);
+    const SettingFieldCustom & getCustomSetting(std::string_view name) const;
+    const SettingFieldCustom * tryGetCustomSetting(std::string_view name) const;
 
     std::conditional_t<Traits::allow_custom_settings, CustomSettingMap, boost::blank> custom_settings_map;
 };
 
 struct BaseSettingsHelpers
 {
-    [[noreturn]] static void throwSettingNotFound(const std::string_view & name);
-    static void warningSettingNotFound(const std::string_view & name);
+    [[noreturn]] static void throwSettingNotFound(std::string_view name);
+    static void warningSettingNotFound(std::string_view name);
 
-    static void writeString(const std::string_view & str, WriteBuffer & out);
+    static void writeString(std::string_view str, WriteBuffer & out);
     static String readString(ReadBuffer & in);
 
     enum Flags : UInt64
@@ -190,7 +190,7 @@ struct BaseSettingsHelpers
 };
 
 template <typename TTraits>
-void BaseSettings<TTraits>::set(const std::string_view & name, const Field & value)
+void BaseSettings<TTraits>::set(std::string_view name, const Field & value)
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -200,7 +200,7 @@ void BaseSettings<TTraits>::set(const std::string_view & name, const Field & val
 }
 
 template <typename TTraits>
-Field BaseSettings<TTraits>::get(const std::string_view & name) const
+Field BaseSettings<TTraits>::get(std::string_view name) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -210,7 +210,7 @@ Field BaseSettings<TTraits>::get(const std::string_view & name) const
 }
 
 template <typename TTraits>
-void BaseSettings<TTraits>::setString(const std::string_view & name, const String & value)
+void BaseSettings<TTraits>::setString(std::string_view name, const String & value)
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -220,7 +220,7 @@ void BaseSettings<TTraits>::setString(const std::string_view & name, const Strin
 }
 
 template <typename TTraits>
-String BaseSettings<TTraits>::getString(const std::string_view & name) const
+String BaseSettings<TTraits>::getString(std::string_view name) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -230,7 +230,7 @@ String BaseSettings<TTraits>::getString(const std::string_view & name) const
 }
 
 template <typename TTraits>
-bool BaseSettings<TTraits>::tryGet(const std::string_view & name, Field & value) const
+bool BaseSettings<TTraits>::tryGet(std::string_view name, Field & value) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -247,7 +247,7 @@ bool BaseSettings<TTraits>::tryGet(const std::string_view & name, Field & value)
 }
 
 template <typename TTraits>
-bool BaseSettings<TTraits>::tryGetString(const std::string_view & name, String & value) const
+bool BaseSettings<TTraits>::tryGetString(std::string_view name, String & value) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -264,7 +264,7 @@ bool BaseSettings<TTraits>::tryGetString(const std::string_view & name, String &
 }
 
 template <typename TTraits>
-bool BaseSettings<TTraits>::isChanged(const std::string_view & name) const
+bool BaseSettings<TTraits>::isChanged(std::string_view name) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -316,20 +316,20 @@ void BaseSettings<TTraits>::resetToDefault()
 }
 
 template <typename TTraits>
-bool BaseSettings<TTraits>::hasBuiltin(const std::string_view & name)
+bool BaseSettings<TTraits>::hasBuiltin(std::string_view name)
 {
     const auto & accessor = Traits::Accessor::instance();
     return (accessor.find(name) != static_cast<size_t>(-1));
 }
 
 template <typename TTraits>
-bool BaseSettings<TTraits>::hasCustom(const std::string_view & name) const
+bool BaseSettings<TTraits>::hasCustom(std::string_view name) const
 {
     return tryGetCustomSetting(name);
 }
 
 template <typename TTraits>
-const char * BaseSettings<TTraits>::getTypeName(const std::string_view & name) const
+const char * BaseSettings<TTraits>::getTypeName(std::string_view name) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -341,7 +341,7 @@ const char * BaseSettings<TTraits>::getTypeName(const std::string_view & name) c
 }
 
 template <typename TTraits>
-const char * BaseSettings<TTraits>::getDescription(const std::string_view & name) const
+const char * BaseSettings<TTraits>::getDescription(std::string_view name) const
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -353,19 +353,19 @@ const char * BaseSettings<TTraits>::getDescription(const std::string_view & name
 }
 
 template <typename TTraits>
-void BaseSettings<TTraits>::checkCanSet(const std::string_view & name, const Field & value)
+void BaseSettings<TTraits>::checkCanSet(std::string_view name, const Field & value)
 {
     castValueUtil(name, value);
 }
 
 template <typename TTraits>
-void BaseSettings<TTraits>::checkCanSetString(const std::string_view & name, const String & str)
+void BaseSettings<TTraits>::checkCanSetString(std::string_view name, const String & str)
 {
     stringToValueUtil(name, str);
 }
 
 template <typename TTraits>
-Field BaseSettings<TTraits>::castValueUtil(const std::string_view & name, const Field & value)
+Field BaseSettings<TTraits>::castValueUtil(std::string_view name, const Field & value)
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -377,7 +377,7 @@ Field BaseSettings<TTraits>::castValueUtil(const std::string_view & name, const
 }
 
 template <typename TTraits>
-String BaseSettings<TTraits>::valueToStringUtil(const std::string_view & name, const Field & value)
+String BaseSettings<TTraits>::valueToStringUtil(std::string_view name, const Field & value)
 {
     const auto & accessor = Traits::Accessor::instance();
     if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@@ -389,7 +389,7 @@ String BaseSettings<TTraits>::valueToStringUtil(const std::string_view & name, c
 }
 
 template <typename TTraits>
-Field BaseSettings<TTraits>::stringToValueUtil(const std::string_view & name, const String & str)
+Field BaseSettings<TTraits>::stringToValueUtil(std::string_view name, const String & str)
 {
     try
     {
@@ -521,7 +521,7 @@ bool operator!=(const BaseSettings<TTraits> & left, const BaseSettings<TTraits>
 }
 
 template <typename TTraits>
-SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_view & name)
+SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(std::string_view name)
 {
     if constexpr (Traits::allow_custom_settings)
     {
@@ -537,7 +537,7 @@ SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_v
 }
 
 template <typename TTraits>
-const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_view & name) const
+const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(std::string_view name) const
 {
     if constexpr (Traits::allow_custom_settings)
     {
@@ -549,7 +549,7 @@ const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::st
 }
 
 template <typename TTraits>
-const SettingFieldCustom * BaseSettings<TTraits>::tryGetCustomSetting(const std::string_view & name) const
+const SettingFieldCustom * BaseSettings<TTraits>::tryGetCustomSetting(std::string_view name) const
 {
     if constexpr (Traits::allow_custom_settings)
     {
@@ -780,7 +780,7 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
         public: \
             static const Accessor & instance(); \
             size_t size() const { return field_infos.size(); } \
-            size_t find(const std::string_view & name) const; \
+            size_t find(std::string_view name) const; \
             const String & getName(size_t index) const { return field_infos[index].name; } \
             const char * getTypeName(size_t index) const { return field_infos[index].type; } \
             const char * getDescription(size_t index) const { return field_infos[index].description; } \
@@ -851,7 +851,7 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
     \
     SETTINGS_TRAITS_NAME::Accessor::Accessor() {} \
     \
-    size_t SETTINGS_TRAITS_NAME::Accessor::find(const std::string_view & name) const \
+    size_t SETTINGS_TRAITS_NAME::Accessor::find(std::string_view name) const \
     { \
         auto it = name_to_index_map.find(name); \
         if (it != name_to_index_map.end()) \

src/Core/Field.cpp

@@ -286,7 +286,7 @@ String Field::dump() const
     return applyVisitor(FieldVisitorDump(), *this);
 }
 
-Field Field::restoreFromDump(const std::string_view & dump_)
+Field Field::restoreFromDump(std::string_view dump_)
 {
     auto show_error = [&dump_]
     {

src/Core/Field.h

@@ -346,7 +346,7 @@ public:
     }
 
     /// Create a string inplace.
-    Field(const std::string_view & str) { create(str.data(), str.size()); } /// NOLINT
+    Field(std::string_view str) { create(str.data(), str.size()); } /// NOLINT
     Field(const String & str) { create(std::string_view{str}); } /// NOLINT
     Field(String && str) { create(std::move(str)); } /// NOLINT
     Field(const char * str) { create(std::string_view{str}); } /// NOLINT
@@ -403,7 +403,7 @@ public:
         return *this;
     }
 
-    Field & operator= (const std::string_view & str);
+    Field & operator= (std::string_view str);
     Field & operator= (const String & str) { return *this = std::string_view{str}; }
     Field & operator= (String && str);
     Field & operator= (const char * str) { return *this = std::string_view{str}; }
@@ -631,7 +631,7 @@ public:
     }
 
     String dump() const;
-    static Field restoreFromDump(const std::string_view & dump_);
+    static Field restoreFromDump(std::string_view dump_);
 
 private:
     std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(Types::Which),
@@ -929,7 +929,7 @@ Field::operator=(T && rhs)
     return *this;
 }
 
-inline Field & Field::operator=(const std::string_view & str)
+inline Field & Field::operator=(std::string_view str)
 {
     if (which != Types::String)
     {

src/Core/Settings.h

@@ -96,6 +96,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
     M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \
     M(UInt64, max_replicated_fetches_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated fetches. Zero means unlimited. Only has meaning at server startup.", 0) \
     M(UInt64, max_replicated_sends_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited. Only has meaning at server startup.", 0) \
+    M(UInt64, max_remote_read_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for read. Zero means unlimited. Only has meaning at server startup.", 0) \
+    M(UInt64, max_remote_write_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for write. Zero means unlimited. Only has meaning at server startup.", 0) \
     M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \
     M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \
     \

src/Core/SettingsFields.cpp

@@ -350,7 +350,7 @@ void SettingFieldURI::readBinary(ReadBuffer & in)
 }
 
 
-void SettingFieldEnumHelpers::writeBinary(const std::string_view & str, WriteBuffer & out)
+void SettingFieldEnumHelpers::writeBinary(std::string_view str, WriteBuffer & out)
 {
     writeStringBinary(str, out);
 }

src/Core/SettingsFields.h

@@ -146,13 +146,13 @@ struct SettingFieldString
     String value;
     bool changed = false;
 
-    explicit SettingFieldString(const std::string_view & str = {}) : value(str) {}
+    explicit SettingFieldString(std::string_view str = {}) : value(str) {}
     explicit SettingFieldString(const String & str) : SettingFieldString(std::string_view{str}) {}
     explicit SettingFieldString(String && str) : value(std::move(str)) {}
     explicit SettingFieldString(const char * str) : SettingFieldString(std::string_view{str}) {}
     explicit SettingFieldString(const Field & f) : SettingFieldString(f.safeGet<const String &>()) {}
 
-    SettingFieldString & operator =(const std::string_view & str) { value = str; changed = true; return *this; }
+    SettingFieldString & operator =(std::string_view str) { value = str; changed = true; return *this; }
     SettingFieldString & operator =(const String & str) { *this = std::string_view{str}; return *this; }
     SettingFieldString & operator =(String && str) { value = std::move(str); changed = true; return *this; }
     SettingFieldString & operator =(const char * str) { *this = std::string_view{str}; return *this; }
@@ -256,7 +256,7 @@ struct SettingFieldEnum
 
 struct SettingFieldEnumHelpers
 {
-    static void writeBinary(const std::string_view & str, WriteBuffer & out);
+    static void writeBinary(std::string_view str, WriteBuffer & out);
     static String readBinary(ReadBuffer & in);
 };
 
@@ -286,7 +286,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
     { \
         using EnumType = ENUM_TYPE; \
         static const String & toString(EnumType value); \
-        static EnumType fromString(const std::string_view & str); \
+        static EnumType fromString(std::string_view str); \
     }; \
     \
     using SettingField##NEW_NAME = SettingFieldEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;
@@ -310,7 +310,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
             ERROR_CODE_FOR_UNEXPECTED_NAME); \
     } \
     \
-    typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(const std::string_view & str) \
+    typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(std::string_view str) \
     { \
         static const std::unordered_map<std::string_view, EnumType> map = [] { \
             std::unordered_map<std::string_view, EnumType> res; \
@@ -430,7 +430,7 @@ void SettingFieldMultiEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
         using EnumType = ENUM_TYPE; \
         static size_t getEnumSize(); \
         static const String & toString(EnumType value); \
-        static EnumType fromString(const std::string_view & str); \
+        static EnumType fromString(std::string_view str); \
     }; \
     \
     using SettingField##NEW_NAME = SettingFieldMultiEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;

src/DataTypes/NestedUtils.cpp

@@ -54,7 +54,7 @@ std::pair<std::string, std::string> splitName(const std::string & name, bool rev
     return {name.substr(0, idx), name.substr(idx + 1)};
 }
 
-std::pair<std::string_view, std::string_view> splitName(const std::string_view & name, bool reverse)
+std::pair<std::string_view, std::string_view> splitName(std::string_view name, bool reverse)
 {
     auto idx = (reverse ? name.find_last_of('.') : name.find_first_of('.'));
     if (idx == std::string::npos || idx == 0 || idx + 1 == name.size())

src/DataTypes/NestedUtils.h

@@ -13,7 +13,7 @@ namespace Nested
 
     /// Splits name of compound identifier by first/last dot (depending on 'reverse' parameter).
     std::pair<std::string, std::string> splitName(const std::string & name, bool reverse = false);
-    std::pair<std::string_view, std::string_view> splitName(const std::string_view & name, bool reverse = false);
+    std::pair<std::string_view, std::string_view> splitName(std::string_view name, bool reverse = false);
 
     /// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot.
     std::string extractTableName(const std::string & nested_name);

src/Dictionaries/FlatDictionary.cpp

@@ -105,7 +105,7 @@ ColumnPtr FlatDictionary::getColumn(
                 getItemsImpl<ValueType, true>(
                     attribute,
                     ids,
-                    [&](size_t row, const StringRef value, bool is_null)
+                    [&](size_t row, StringRef value, bool is_null)
                     {
                         (*vec_null_map_to)[row] = is_null;
                         out->insertData(value.data, value.size);
@@ -115,7 +115,7 @@ ColumnPtr FlatDictionary::getColumn(
                 getItemsImpl<ValueType, false>(
                     attribute,
                     ids,
-                    [&](size_t, const StringRef value, bool) { out->insertData(value.data, value.size); },
+                    [&](size_t, StringRef value, bool) { out->insertData(value.data, value.size); },
                     default_value_extractor);
         }
         else

src/Dictionaries/HashedArrayDictionary.cpp

@@ -585,7 +585,7 @@ ColumnPtr HashedArrayDictionary<dictionary_key_type>::getAttributeColumn(
                 getItemsImpl<ValueType, true>(
                     attribute,
                     keys_object,
-                    [&](size_t row, const StringRef value, bool is_null)
+                    [&](size_t row, StringRef value, bool is_null)
                     {
                         (*vec_null_map_to)[row] = is_null;
                         out->insertData(value.data, value.size);
@@ -595,7 +595,7 @@ ColumnPtr HashedArrayDictionary<dictionary_key_type>::getAttributeColumn(
                 getItemsImpl<ValueType, false>(
                     attribute,
                     keys_object,
-                    [&](size_t, const StringRef value, bool) { out->insertData(value.data, value.size); },
+                    [&](size_t, StringRef value, bool) { out->insertData(value.data, value.size); },
                     default_value_extractor);
         }
         else

src/Dictionaries/HashedDictionary.cpp

@@ -117,7 +117,7 @@ ColumnPtr HashedDictionary<dictionary_key_type, sparse>::getColumn(
                 getItemsImpl<ValueType, true>(
                     attribute,
                     extractor,
-                    [&](size_t row, const StringRef value, bool is_null)
+                    [&](size_t row, StringRef value, bool is_null)
                     {
                         (*vec_null_map_to)[row] = is_null;
                         out->insertData(value.data, value.size);
@@ -127,7 +127,7 @@ ColumnPtr HashedDictionary<dictionary_key_type, sparse>::getColumn(
                 getItemsImpl<ValueType, false>(
                     attribute,
                     extractor,
-                    [&](size_t, const StringRef value, bool) { out->insertData(value.data, value.size); },
+                    [&](size_t, StringRef value, bool) { out->insertData(value.data, value.size); },
                     default_value_extractor);
         }
         else

src/Dictionaries/IPAddressDictionary.cpp

@@ -261,7 +261,7 @@ ColumnPtr IPAddressDictionary::getColumn(
             getItemsImpl<ValueType>(
                 attribute,
                 key_columns,
-                [&](const size_t, const StringRef value) { out->insertData(value.data, value.size); },
+                [&](const size_t, StringRef value) { out->insertData(value.data, value.size); },
                 default_value_extractor);
         }
         else

src/Dictionaries/RangeHashedDictionary.cpp

@@ -151,7 +151,7 @@ ColumnPtr RangeHashedDictionary<dictionary_key_type>::getColumn(
                 getItemsImpl<ValueType, true>(
                     attribute,
                     modified_key_columns,
-                    [&](size_t row, const StringRef value, bool is_null)
+                    [&](size_t row, StringRef value, bool is_null)
                     {
                         (*vec_null_map_to)[row] = is_null;
                         out->insertData(value.data, value.size);
@@ -161,7 +161,7 @@ ColumnPtr RangeHashedDictionary<dictionary_key_type>::getColumn(
                 getItemsImpl<ValueType, false>(
                     attribute,
                     modified_key_columns,
-                    [&](size_t, const StringRef value, bool)
+                    [&](size_t, StringRef value, bool)
                     {
                         out->insertData(value.data, value.size);
                     },
@@ -255,7 +255,7 @@ ColumnPtr RangeHashedDictionary<dictionary_key_type>::getColumnInternal(
                 getItemsInternalImpl<ValueType, true>(
                     attribute,
                     key_to_index,
-                    [&](size_t row, const StringRef value, bool is_null)
+                    [&](size_t row, StringRef value, bool is_null)
                     {
                         (*vec_null_map_to)[row] = is_null;
                         out->insertData(value.data, value.size);
@@ -264,7 +264,7 @@ ColumnPtr RangeHashedDictionary<dictionary_key_type>::getColumnInternal(
                 getItemsInternalImpl<ValueType, false>(
                     attribute,
                     key_to_index,
-                    [&](size_t, const StringRef value, bool)
+                    [&](size_t, StringRef value, bool)
                     {
                         out->insertData(value.data, value.size);
                     });

src/Disks/IO/ReadBufferFromRemoteFSGather.cpp

@@ -90,7 +90,6 @@ SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementation
         settings,
         max_single_read_retries,
         max_single_download_retries,
-        settings.remote_fs_buffer_size,
         /* use_external_buffer */true,
         read_until_position);
 }

src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp

@@ -67,8 +67,8 @@ std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObject( /// NOLI
     auto settings_ptr = settings.get();
 
     return std::make_unique<ReadBufferFromAzureBlobStorage>(
-        client.get(), object.absolute_path, read_settings, settings_ptr->max_single_read_retries,
-        settings_ptr->max_single_download_retries, read_settings.remote_fs_buffer_size);
+        client.get(), object.absolute_path, patchSettings(read_settings), settings_ptr->max_single_read_retries,
+        settings_ptr->max_single_download_retries);
 }
 
 std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOLINT
@@ -77,18 +77,19 @@ std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOL
     std::optional<size_t>,
     std::optional<size_t>) const
 {
+    ReadSettings disk_read_settings = patchSettings(read_settings);
     auto settings_ptr = settings.get();
     auto reader_impl = std::make_unique<ReadBufferFromAzureBlobStorageGather>(
         client.get(),
         objects,
         settings_ptr->max_single_read_retries,
         settings_ptr->max_single_download_retries,
-        read_settings);
+        disk_read_settings);
 
-    if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
+    if (disk_read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
     {
         auto reader = getThreadPoolReader();
-        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, read_settings, std::move(reader_impl));
+        return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, disk_read_settings, std::move(reader_impl));
     }
     else
     {
@@ -104,7 +105,7 @@ std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NO
     std::optional<ObjectAttributes>,
     FinalizeCallback && finalize_callback,
     size_t buf_size,
-    const WriteSettings &)
+    const WriteSettings & write_settings)
 {
     if (mode != WriteMode::Rewrite)
         throw Exception("Azure storage doesn't support append", ErrorCodes::UNSUPPORTED_METHOD);
@@ -113,7 +114,8 @@ std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NO
         client.get(),
         object.absolute_path,
         settings.get()->max_single_part_upload_size,
-        buf_size);
+        buf_size,
+        patchSettings(write_settings));
 
     return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(finalize_callback), object.absolute_path);
 }
@@ -207,7 +209,7 @@ void AzureObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguratio
 {
     auto new_settings = getAzureBlobStorageSettings(config, config_prefix, context);
     settings.set(std::move(new_settings));
-
+    applyRemoteThrottlingSettings(context);
     /// We don't update client
 }
 

src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.cpp

@@ -11,7 +11,6 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int NOT_IMPLEMENTED;
-    extern const int FS_METADATA_ERROR;
 }
 
 FakeMetadataStorageFromDisk::FakeMetadataStorageFromDisk(
@@ -114,85 +113,6 @@ const IMetadataStorage & FakeMetadataStorageFromDiskTransaction::getStorageForNo
     return metadata_storage;
 }
 
-void FakeMetadataStorageFromDiskTransaction::addOperation(MetadataOperationPtr && operation)
-{
-    if (state != MetadataFromDiskTransactionState::PREPARING)
-        throw Exception(
-            ErrorCodes::FS_METADATA_ERROR,
-            "Cannot add operations to transaction in {} state, it should be in {} state",
-            toString(state), toString(MetadataFromDiskTransactionState::PREPARING));
-
-    operations.emplace_back(std::move(operation));
-}
-
-void FakeMetadataStorageFromDiskTransaction::commit()
-{
-    if (state != MetadataFromDiskTransactionState::PREPARING)
-        throw Exception(
-            ErrorCodes::FS_METADATA_ERROR,
-            "Cannot commit transaction in {} state, it should be in {} state",
-            toString(state), toString(MetadataFromDiskTransactionState::PREPARING));
-
-    {
-        std::unique_lock lock(metadata_storage.metadata_mutex);
-        for (size_t i = 0; i < operations.size(); ++i)
-        {
-            try
-            {
-                operations[i]->execute();
-            }
-            catch (Exception & ex)
-            {
-                tryLogCurrentException(__PRETTY_FUNCTION__);
-                ex.addMessage(fmt::format("While committing metadata operation #{}", i));
-                state = MetadataFromDiskTransactionState::FAILED;
-                rollback(i);
-                throw;
-            }
-        }
-    }
-
-    /// Do it in "best effort" mode
-    for (size_t i = 0; i < operations.size(); ++i)
-    {
-        try
-        {
-            operations[i]->finalize();
-        }
-        catch (...)
-        {
-            tryLogCurrentException(__PRETTY_FUNCTION__, fmt::format("Failed to finalize operation #{}", i));
-        }
-    }
-
-    state = MetadataFromDiskTransactionState::COMMITTED;
-}
-
-void FakeMetadataStorageFromDiskTransaction::rollback(size_t until_pos)
-{
-    /// Otherwise everything is alright
-    if (state == MetadataFromDiskTransactionState::FAILED)
-    {
-        for (int64_t i = until_pos; i >= 0; --i)
-        {
-            try
-            {
-                operations[i]->undo();
-            }
-            catch (Exception & ex)
-            {
-                state = MetadataFromDiskTransactionState::PARTIALLY_ROLLED_BACK;
-                ex.addMessage(fmt::format("While rolling back operation #{}", i));
-                throw;
-            }
-        }
-    }
-    else
-    {
-        /// Nothing to do, transaction committed or not even started to commit
-    }
-}
-
 void FakeMetadataStorageFromDiskTransaction::writeStringToFile(const std::string & path, const std::string & data)
 {
     auto wb = disk->writeFile(path);

src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.h

@@ -65,12 +65,6 @@ private:
     const FakeMetadataStorageFromDisk & metadata_storage;
 
     std::vector<MetadataOperationPtr> operations;
-    MetadataFromDiskTransactionState state{MetadataFromDiskTransactionState::PREPARING};
-
-    void addOperation(MetadataOperationPtr && operation);
-
-    void rollback(size_t until_pos);
-
 public:
     FakeMetadataStorageFromDiskTransaction(
         const FakeMetadataStorageFromDisk & metadata_storage_, DiskPtr disk_)
@@ -82,7 +76,7 @@ public:
 
     const IMetadataStorage & getStorageForNonTransactionalReads() const final;
 
-    void commit() final;
+    void commit() final {}
 
     void writeStringToFile(const std::string & path, const std::string & data) override;
 

src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp

@@ -52,7 +52,7 @@ std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObject( /// NOLIN
     std::optional<size_t>,
     std::optional<size_t>) const
 {
-    return std::make_unique<ReadBufferFromHDFS>(object.absolute_path, object.absolute_path, config, read_settings);
+    return std::make_unique<ReadBufferFromHDFS>(object.absolute_path, object.absolute_path, config, patchSettings(read_settings));
 }
 
 std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObjects( /// NOLINT
@@ -61,7 +61,7 @@ std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObjects( /// NOLI
     std::optional<size_t>,
     std::optional<size_t>) const
 {
-    auto hdfs_impl = std::make_unique<ReadBufferFromHDFSGather>(config, objects, read_settings);
+    auto hdfs_impl = std::make_unique<ReadBufferFromHDFSGather>(config, objects, patchSettings(read_settings));
     auto buf = std::make_unique<ReadIndirectBufferFromRemoteFS>(std::move(hdfs_impl));
     return std::make_unique<SeekAvoidingReadBuffer>(std::move(buf), settings->min_bytes_for_seek);
 }
@@ -72,7 +72,7 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
     std::optional<ObjectAttributes> attributes,
     FinalizeCallback && finalize_callback,
     size_t buf_size,
-    const WriteSettings &)
+    const WriteSettings & write_settings)
 {
     if (attributes.has_value())
         throw Exception(
@@ -81,7 +81,7 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
 
     /// Single O_WRONLY in libhdfs adds O_TRUNC
     auto hdfs_buffer = std::make_unique<WriteBufferFromHDFS>(
-        object.absolute_path, config, settings->replication, buf_size,
+        object.absolute_path, config, settings->replication, patchSettings(write_settings), buf_size,
         mode == WriteMode::Rewrite ? O_WRONLY : O_WRONLY | O_APPEND);
 
     return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(hdfs_buffer), std::move(finalize_callback), object.absolute_path);
@@ -155,8 +155,9 @@ void HDFSObjectStorage::copyObject( /// NOLINT
 }
 
 
-void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr)
+void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr context)
 {
+    applyRemoteThrottlingSettings(context);
 }
 
 std::unique_ptr<IObjectStorage> HDFSObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr)

src/Disks/ObjectStorages/IObjectStorage.cpp

@@ -47,4 +47,27 @@ std::string IObjectStorage::getCacheBasePath() const
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getCacheBasePath() is not implemented for {}", getName());
 }
 
+void IObjectStorage::applyRemoteThrottlingSettings(ContextPtr context)
+{
+    std::unique_lock lock{throttlers_mutex};
+    remote_read_throttler = context->getRemoteReadThrottler();
+    remote_write_throttler = context->getRemoteWriteThrottler();
+}
+
+ReadSettings IObjectStorage::patchSettings(const ReadSettings & read_settings) const
+{
+    std::unique_lock lock{throttlers_mutex};
+    ReadSettings settings{read_settings};
+    settings.remote_throttler = remote_read_throttler;
+    return settings;
+}
+
+WriteSettings IObjectStorage::patchSettings(const WriteSettings & write_settings) const
+{
+    std::unique_lock lock{throttlers_mutex};
+    WriteSettings settings{write_settings};
+    settings.remote_throttler = remote_write_throttler;
+    return settings;
+}
+
 }

src/Disks/ObjectStorages/IObjectStorage.h

@@ -3,6 +3,7 @@
 #include <filesystem>
 #include <string>
 #include <map>
+#include <mutex>
 #include <optional>
 
 #include <Poco/Timestamp.h>
@@ -166,6 +167,19 @@ public:
     virtual void removeCacheIfExists(const std::string & /* path */) {}
 
     virtual bool supportsCache() const { return false; }
+
+protected:
+    /// Should be called from implementation of applyNewSettings()
+    void applyRemoteThrottlingSettings(ContextPtr context);
+
+    /// Should be used by implementation of read* and write* methods
+    ReadSettings patchSettings(const ReadSettings & read_settings) const;
+    WriteSettings patchSettings(const WriteSettings & write_settings) const;
+
+private:
+    mutable std::mutex throttlers_mutex;
+    ThrottlerPtr remote_read_throttler;
+    ThrottlerPtr remote_write_throttler;
 };
 
 using ObjectStoragePtr = std::shared_ptr<IObjectStorage>;

src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp

@@ -87,6 +87,14 @@ DiskObjectStorageMetadataPtr MetadataStorageFromDisk::readMetadataUnlocked(const
     return metadata;
 }
 
+DiskObjectStorageMetadataPtr MetadataStorageFromDisk::readMetadataUnlocked(const std::string & path, std::unique_lock<std::shared_mutex> &) const
+{
+    auto metadata = std::make_unique<DiskObjectStorageMetadata>(disk->getPath(), object_storage_root_path, path);
+    auto str = readFileToString(path);
+    metadata->deserializeFromString(str);
+    return metadata;
+}
+
 DiskObjectStorageMetadataPtr MetadataStorageFromDisk::readMetadata(const std::string & path) const
 {
     std::shared_lock lock(metadata_mutex);
@@ -112,13 +120,7 @@ std::unordered_map<String, String> MetadataStorageFromDisk::getSerializedMetadat
 
 void MetadataStorageFromDiskTransaction::createHardLink(const std::string & path_from, const std::string & path_to)
 {
-    auto metadata = metadata_storage.readMetadata(path_from);
-
-    metadata->incrementRefCount();
-
-    writeStringToFile(path_from, metadata->serializeToString());
-
-    addOperation(std::make_unique<CreateHardlinkOperation>(path_from, path_to, *metadata_storage.getDisk()));
+    addOperation(std::make_unique<CreateHardlinkOperation>(path_from, path_to, *metadata_storage.disk, metadata_storage));
 }
 
 MetadataTransactionPtr MetadataStorageFromDisk::createTransaction() const
@@ -177,12 +179,12 @@ void MetadataStorageFromDiskTransaction::commit()
             toString(state), toString(MetadataFromDiskTransactionState::PREPARING));
 
     {
-        std::lock_guard lock(metadata_storage.metadata_mutex);
+        std::unique_lock lock(metadata_storage.metadata_mutex);
         for (size_t i = 0; i < operations.size(); ++i)
         {
             try
             {
-                operations[i]->execute();
+                operations[i]->execute(lock);
             }
             catch (Exception & ex)
             {
@@ -316,29 +318,12 @@ void MetadataStorageFromDiskTransaction::createMetadataFile(const std::string &
 
 void MetadataStorageFromDiskTransaction::addBlobToMetadata(const std::string & path, const std::string & blob_name, uint64_t size_in_bytes)
 {
-    DiskObjectStorageMetadataPtr metadata;
-    if (metadata_storage.exists(path))
-    {
-        metadata = metadata_storage.readMetadata(path);
-        metadata->addObject(blob_name, size_in_bytes);
-        writeStringToFile(path, metadata->serializeToString());
-    }
-    else
-    {
-        createMetadataFile(path, blob_name, size_in_bytes);
-    }
+    addOperation(std::make_unique<AddBlobOperation>(path, blob_name, metadata_storage.object_storage_root_path, size_in_bytes, *metadata_storage.disk, metadata_storage));
 }
 
 void MetadataStorageFromDiskTransaction::unlinkMetadata(const std::string & path)
 {
-    auto metadata = metadata_storage.readMetadata(path);
-    uint32_t ref_count = metadata->getRefCount();
-    if (ref_count != 0)
-    {
-        metadata->decrementRefCount();
-        writeStringToFile(path, metadata->serializeToString());
-    }
-    unlinkFile(path);
+    addOperation(std::make_unique<UnlinkMetadataFileOperation>(path, *metadata_storage.disk, metadata_storage));
 }
 
 }

src/Disks/ObjectStorages/MetadataStorageFromDisk.h

@@ -55,9 +55,9 @@ public:
 
     std::string getObjectStorageRootPath() const override { return object_storage_root_path; }
 
-private:
     DiskObjectStorageMetadataPtr readMetadata(const std::string & path) const;
 
+    DiskObjectStorageMetadataPtr readMetadataUnlocked(const std::string & path, std::unique_lock<std::shared_mutex> & lock) const;
     DiskObjectStorageMetadataPtr readMetadataUnlocked(const std::string & path, std::shared_lock<std::shared_mutex> & lock) const;
 };
 

src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.cpp

@@ -1,4 +1,5 @@
 #include <Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.h>
+#include <Disks/ObjectStorages/MetadataStorageFromDisk.h>
 #include <Disks/IDisk.h>
 #include <Common/getRandomASCIIString.h>
 #include <IO/WriteHelpers.h>
@@ -24,7 +25,7 @@ SetLastModifiedOperation::SetLastModifiedOperation(const std::string & path_, Po
 {
 }
 
-void SetLastModifiedOperation::execute()
+void SetLastModifiedOperation::execute(std::unique_lock<std::shared_mutex> &)
 {
     old_timestamp = disk.getLastModified(path);
     disk.setLastModified(path, new_timestamp);
@@ -41,7 +42,7 @@ UnlinkFileOperation::UnlinkFileOperation(const std::string & path_, IDisk & disk
 {
 }
 
-void UnlinkFileOperation::execute()
+void UnlinkFileOperation::execute(std::unique_lock<std::shared_mutex> &)
 {
     auto buf = disk.readFile(path);
     readStringUntilEOF(prev_data, *buf);
@@ -61,7 +62,7 @@ CreateDirectoryOperation::CreateDirectoryOperation(const std::string & path_, ID
 {
 }
 
-void CreateDirectoryOperation::execute()
+void CreateDirectoryOperation::execute(std::unique_lock<std::shared_mutex> &)
 {
     disk.createDirectory(path);
 }
@@ -77,7 +78,7 @@ CreateDirectoryRecursiveOperation::CreateDirectoryRecursiveOperation(const std::
 {
 }
 
-void CreateDirectoryRecursiveOperation::execute()
+void CreateDirectoryRecursiveOperation::execute(std::unique_lock<std::shared_mutex> &)
 {
     namespace fs = std::filesystem;
     fs::path p(path);
@@ -104,7 +105,7 @@ RemoveDirectoryOperation::RemoveDirectoryOperation(const std::string & path_, ID
 {
 }
 
-void RemoveDirectoryOperation::execute()
+void RemoveDirectoryOperation::execute(std::unique_lock<std::shared_mutex> &)
 {
     disk.removeDirectory(path);
 }
@@ -121,7 +122,7 @@ RemoveRecursiveOperation::RemoveRecursiveOperation(const std::string & path_, ID
 {
 }
 
-void RemoveRecursiveOperation:: execute()
+void RemoveRecursiveOperation::execute(std::unique_lock<std::shared_mutex> &)
 {
     if (disk.isFile(path))
         disk.moveFile(path, temp_path);
@@ -146,20 +147,31 @@ void RemoveRecursiveOperation::finalize()
         disk.removeRecursive(path);
 }
 
-CreateHardlinkOperation::CreateHardlinkOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_)
+CreateHardlinkOperation::CreateHardlinkOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_, const MetadataStorageFromDisk & metadata_storage_)
     : path_from(path_from_)
     , path_to(path_to_)
     , disk(disk_)
+    , metadata_storage(metadata_storage_)
 {
 }
 
-void CreateHardlinkOperation::execute()
+void CreateHardlinkOperation::execute(std::unique_lock<std::shared_mutex> & lock)
 {
+    auto metadata = metadata_storage.readMetadataUnlocked(path_from, lock);
+
+    metadata->incrementRefCount();
+
+    write_operation = std::make_unique<WriteFileOperation>(path_from, disk, metadata->serializeToString());
+
+    write_operation->execute(lock);
+
     disk.createHardLink(path_from, path_to);
 }
 
 void CreateHardlinkOperation::undo()
 {
+    if (write_operation)
+        write_operation->undo();
     disk.removeFile(path_to);
 }
 
@@ -170,7 +182,7 @@ MoveFileOperation::MoveFileOperation(const std::string & path_from_, const std::
 {
 }
 
-void MoveFileOperation::execute()
+void MoveFileOperation::execute(std::unique_lock<std::shared_mutex> &)
 {
     disk.moveFile(path_from, path_to);
 }
@@ -187,7 +199,7 @@ MoveDirectoryOperation::MoveDirectoryOperation(const std::string & path_from_, c
 {
 }
 
-void MoveDirectoryOperation::execute()
+void MoveDirectoryOperation::execute(std::unique_lock<std::shared_mutex> &)
 {
     disk.moveDirectory(path_from, path_to);
 }
@@ -197,7 +209,6 @@ void MoveDirectoryOperation::undo()
     disk.moveDirectory(path_to, path_from);
 }
 
-
 ReplaceFileOperation::ReplaceFileOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_)
     : path_from(path_from_)
     , path_to(path_to_)
@@ -206,7 +217,7 @@ ReplaceFileOperation::ReplaceFileOperation(const std::string & path_from_, const
 {
 }
 
-void ReplaceFileOperation::execute()
+void ReplaceFileOperation::execute(std::unique_lock<std::shared_mutex> &)
 {
     if (disk.exists(path_to))
         disk.moveFile(path_to, temp_path_to);
@@ -232,7 +243,7 @@ WriteFileOperation::WriteFileOperation(const std::string & path_, IDisk & disk_,
 {
 }
 
-void WriteFileOperation::execute()
+void WriteFileOperation::execute(std::unique_lock<std::shared_mutex> &)
 {
     if (disk.exists(path))
     {
@@ -258,4 +269,62 @@ void WriteFileOperation::undo()
     }
 }
 
+void AddBlobOperation::execute(std::unique_lock<std::shared_mutex> & metadata_lock)
+{
+    DiskObjectStorageMetadataPtr metadata;
+    if (metadata_storage.exists(path))
+        metadata = metadata_storage.readMetadataUnlocked(path, metadata_lock);
+    else
+        metadata = std::make_unique<DiskObjectStorageMetadata>(disk.getPath(), root_path, path);
+
+    metadata->addObject(blob_name, size_in_bytes);
+
+    write_operation = std::make_unique<WriteFileOperation>(path, disk, metadata->serializeToString());
+
+    write_operation->execute(metadata_lock);
+}
+
+void AddBlobOperation::undo()
+{
+    if (write_operation)
+        write_operation->undo();
+}
+
+void UnlinkMetadataFileOperation::execute(std::unique_lock<std::shared_mutex> & metadata_lock)
+{
+    auto metadata = metadata_storage.readMetadataUnlocked(path, metadata_lock);
+    uint32_t ref_count = metadata->getRefCount();
+    if (ref_count != 0)
+    {
+        metadata->decrementRefCount();
+        write_operation = std::make_unique<WriteFileOperation>(path, disk, metadata->serializeToString());
+        write_operation->execute(metadata_lock);
+    }
+    unlink_operation = std::make_unique<UnlinkFileOperation>(path, disk);
+    unlink_operation->execute(metadata_lock);
+}
+
+void UnlinkMetadataFileOperation::undo()
+{
+    if (write_operation)
+        write_operation->undo();
+
+    if (unlink_operation)
+        unlink_operation->undo();
+}
+
+void SetReadonlyFileOperation::execute(std::unique_lock<std::shared_mutex> & metadata_lock)
+{
+    auto metadata = metadata_storage.readMetadataUnlocked(path, metadata_lock);
+    metadata->setReadOnly();
+    write_operation = std::make_unique<WriteFileOperation>(path, disk, metadata->serializeToString());
+    write_operation->execute(metadata_lock);
+}
+
+void SetReadonlyFileOperation::undo()
+{
+    if (write_operation)
+        write_operation->undo();
+}
+
 }

src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.h

@@ -4,6 +4,7 @@
 
 namespace DB
 {
+class MetadataStorageFromDisk;
 class IDisk;
 
 /**
@@ -12,7 +13,7 @@ class IDisk;
 
 struct IMetadataOperation
 {
-    virtual void execute() = 0;
+    virtual void execute(std::unique_lock<std::shared_mutex> & metadata_lock) = 0;
     virtual void undo() = 0;
     virtual void finalize() {}
     virtual ~IMetadataOperation() = default;
@@ -25,7 +26,7 @@ struct SetLastModifiedOperation final : public IMetadataOperation
 {
     SetLastModifiedOperation(const std::string & path_, Poco::Timestamp new_timestamp_, IDisk & disk_);
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
 
@@ -41,7 +42,7 @@ struct UnlinkFileOperation final : public IMetadataOperation
 {
     UnlinkFileOperation(const std::string & path_, IDisk & disk_);
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
 
@@ -56,7 +57,7 @@ struct CreateDirectoryOperation final : public IMetadataOperation
 {
     CreateDirectoryOperation(const std::string & path_, IDisk & disk_);
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
 
@@ -70,7 +71,7 @@ struct CreateDirectoryRecursiveOperation final : public IMetadataOperation
 {
     CreateDirectoryRecursiveOperation(const std::string & path_, IDisk & disk_);
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
 
@@ -85,7 +86,7 @@ struct RemoveDirectoryOperation final : public IMetadataOperation
 {
     RemoveDirectoryOperation(const std::string & path_, IDisk & disk_);
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
 
@@ -98,7 +99,7 @@ struct RemoveRecursiveOperation final : public IMetadataOperation
 {
     RemoveRecursiveOperation(const std::string & path_, IDisk & disk_);
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
 
@@ -110,12 +111,30 @@ private:
     std::string temp_path;
 };
 
+struct WriteFileOperation final : public IMetadataOperation
+{
+    WriteFileOperation(const std::string & path_, IDisk & disk_, const std::string & data_);
+
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
+
+    void undo() override;
+private:
+    std::string path;
+    IDisk & disk;
+    std::string data;
+    bool existed = false;
+    std::string prev_data;
+};
 
 struct CreateHardlinkOperation final : public IMetadataOperation
 {
-    CreateHardlinkOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_);
+    CreateHardlinkOperation(
+        const std::string & path_from_,
+        const std::string & path_to_,
+        IDisk & disk_,
+        const MetadataStorageFromDisk & metadata_storage_);
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
 
@@ -123,6 +142,8 @@ private:
     std::string path_from;
     std::string path_to;
     IDisk & disk;
+    std::unique_ptr<WriteFileOperation> write_operation;
+    const MetadataStorageFromDisk & metadata_storage;
 };
 
 
@@ -130,7 +151,7 @@ struct MoveFileOperation final : public IMetadataOperation
 {
     MoveFileOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_);
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
 
@@ -145,7 +166,7 @@ struct MoveDirectoryOperation final : public IMetadataOperation
 {
     MoveDirectoryOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_);
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
 
@@ -160,7 +181,7 @@ struct ReplaceFileOperation final : public IMetadataOperation
 {
     ReplaceFileOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_);
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
 
@@ -173,20 +194,86 @@ private:
     std::string temp_path_to;
 };
 
-
-struct WriteFileOperation final : public IMetadataOperation
+struct AddBlobOperation final : public IMetadataOperation
 {
-    WriteFileOperation(const std::string & path_, IDisk & disk_, const std::string & data_);
+    AddBlobOperation(
+        const std::string & path_,
+        const std::string & blob_name_,
+        const std::string & root_path_,
+        uint64_t size_in_bytes_,
+        IDisk & disk_,
+        const MetadataStorageFromDisk & metadata_storage_)
+        : path(path_)
+        , blob_name(blob_name_)
+        , root_path(root_path_)
+        , size_in_bytes(size_in_bytes_)
+        , disk(disk_)
+        , metadata_storage(metadata_storage_)
+    {}
 
-    void execute() override;
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
 
     void undo() override;
+
+private:
+    std::string path;
+    std::string blob_name;
+    std::string root_path;
+    uint64_t size_in_bytes;
+    IDisk & disk;
+    const MetadataStorageFromDisk & metadata_storage;
+
+    std::unique_ptr<WriteFileOperation> write_operation;
+};
+
+
+struct UnlinkMetadataFileOperation final : public IMetadataOperation
+{
+    UnlinkMetadataFileOperation(
+        const std::string & path_,
+        IDisk & disk_,
+        const MetadataStorageFromDisk & metadata_storage_)
+        : path(path_)
+        , disk(disk_)
+        , metadata_storage(metadata_storage_)
+    {
+    }
+
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
+
+    void undo() override;
+
 private:
     std::string path;
     IDisk & disk;
-    std::string data;
-    bool existed = false;
-    std::string prev_data;
+    const MetadataStorageFromDisk & metadata_storage;
+
+    std::unique_ptr<WriteFileOperation> write_operation;
+    std::unique_ptr<UnlinkFileOperation> unlink_operation;
+};
+
+struct SetReadonlyFileOperation final : public IMetadataOperation
+{
+    SetReadonlyFileOperation(
+        const std::string & path_,
+        IDisk & disk_,
+        const MetadataStorageFromDisk & metadata_storage_)
+        : path(path_)
+        , disk(disk_)
+        , metadata_storage(metadata_storage_)
+    {
+    }
+
+    void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
+
+    void undo() override;
+
+private:
+    std::string path;
+    IDisk & disk;
+    const MetadataStorageFromDisk & metadata_storage;
+
+    std::unique_ptr<WriteFileOperation> write_operation;
 };
 
 }

src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp

@@ -139,14 +139,7 @@ std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObjects( /// NOLINT
 {
     assert(!objects[0].getPathKeyForCache().empty());
 
-    ReadSettings disk_read_settings{read_settings};
-    if (cache)
-    {
-        if (IFileCache::isReadOnly())
-            disk_read_settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = true;
-
-        disk_read_settings.remote_fs_cache = cache;
-    }
+    ReadSettings disk_read_settings = patchSettings(read_settings);
 
     auto settings_ptr = s3_settings.get();
 
@@ -183,10 +176,9 @@ std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObject( /// NOLINT
         object.absolute_path,
         version_id,
         settings_ptr->s3_settings.max_single_read_retries,
-        read_settings);
+        patchSettings(read_settings));
 }
 
-
 std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLINT
     const StoredObject & object,
     WriteMode mode, // S3 doesn't support append, only rewrite
@@ -195,6 +187,8 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
     size_t buf_size,
     const WriteSettings & write_settings)
 {
+    WriteSettings disk_write_settings = IObjectStorage::patchSettings(write_settings);
+
     if (mode != WriteMode::Rewrite)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "S3 doesn't support append to files");
 
@@ -211,6 +205,7 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
         attributes,
         buf_size,
         threadPoolCallbackRunner(getThreadPoolWriter()),
+        disk_write_settings,
         cache_on_write ? cache : nullptr);
 
 
@@ -485,6 +480,19 @@ void S3ObjectStorage::copyObject( // NOLINT
     }
 }
 
+ReadSettings S3ObjectStorage::patchSettings(const ReadSettings & read_settings) const
+{
+    ReadSettings settings{read_settings};
+    if (cache)
+    {
+        if (IFileCache::isReadOnly())
+            settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = true;
+
+        settings.remote_fs_cache = cache;
+    }
+    return IObjectStorage::patchSettings(settings);
+}
+
 void S3ObjectStorage::setNewSettings(std::unique_ptr<S3ObjectStorageSettings> && s3_settings_)
 {
     s3_settings.set(std::move(s3_settings_));
@@ -517,6 +525,7 @@ void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &
 {
     s3_settings.set(getSettings(config, config_prefix, context));
     client.set(getClient(config, config_prefix, context));
+    applyRemoteThrottlingSettings(context);
 }
 
 std::unique_ptr<IObjectStorage> S3ObjectStorage::cloneObjectStorage(

src/Disks/ObjectStorages/S3/S3ObjectStorage.h

@@ -142,6 +142,8 @@ public:
     String getCacheBasePath() const override;
 
 private:
+    ReadSettings patchSettings(const ReadSettings & read_settings) const;
+
     void setNewSettings(std::unique_ptr<S3ObjectStorageSettings> && s3_settings_);
 
     void setNewClient(std::unique_ptr<Aws::S3::S3Client> && client_);

src/Formats/ProtobufSerializer.cpp

@@ -77,18 +77,18 @@ namespace
             return convertChar(c1) == convertChar(c2);
         }
 
-        static bool equals(const std::string_view & s1, const std::string_view & s2)
+        static bool equals(std::string_view s1, std::string_view s2)
         {
             return (s1.length() == s2.length())
                 && std::equal(s1.begin(), s1.end(), s2.begin(), [](char c1, char c2) { return convertChar(c1) == convertChar(c2); });
         }
 
-        static bool less(const std::string_view & s1, const std::string_view & s2)
+        static bool less(std::string_view s1, std::string_view s2)
         {
             return std::lexicographical_compare(s1.begin(), s1.end(), s2.begin(), s2.end(), [](char c1, char c2) { return convertChar(c1) < convertChar(c2); });
         }
 
-        static bool startsWith(const std::string_view & s1, const std::string_view & s2)
+        static bool startsWith(std::string_view s1, std::string_view s2)
         {
             return (s1.length() >= s2.length()) && equals(s1.substr(0, s2.length()), s2);
         }
@@ -195,7 +195,7 @@ namespace
     {
     protected:
         ProtobufSerializerSingleValue(
-            const std::string_view & column_name_,
+            std::string_view column_name_,
             const FieldDescriptor & field_descriptor_,
             const ProtobufReaderOrWriter & reader_or_writer_)
             : column_name(column_name_)
@@ -264,7 +264,7 @@ namespace
             return reader->readFixed<FieldType>();
         }
 
-        void writeStr(const std::string_view & str)
+        void writeStr(std::string_view str)
         {
             if (!str.empty() || !skip_zero_or_empty)
                 writer->writeString(field_tag, str);
@@ -274,7 +274,7 @@ namespace
         void readStrAndAppend(PaddedPODArray<UInt8> & str) { reader->readStringAndAppend(str); }
 
         template <typename DestType>
-        DestType parseFromStr(const std::string_view & str) const
+        DestType parseFromStr(std::string_view str) const
         {
             try
             {
@@ -307,7 +307,7 @@ namespace
             return result;
         }
 
-        [[noreturn]] void incompatibleColumnType(const std::string_view & column_type) const
+        [[noreturn]] void incompatibleColumnType(std::string_view column_type) const
         {
             throw Exception(
                 ErrorCodes::DATA_TYPE_INCOMPATIBLE_WITH_PROTOBUF_FIELD,
@@ -318,7 +318,7 @@ namespace
                 field_descriptor.type_name());
         }
 
-        [[noreturn]] void cannotConvertValue(const std::string_view & src_value, const std::string_view & src_type_name, const std::string_view & dest_type_name) const
+        [[noreturn]] void cannotConvertValue(std::string_view src_value, std::string_view src_type_name, std::string_view dest_type_name) const
         {
             throw Exception(
                 "Could not convert value '" + String{src_value} + "' from type " + String{src_type_name} + " to type "
@@ -351,7 +351,7 @@ namespace
     public:
         using ColumnType = ColumnVector<NumberType>;
 
-        ProtobufSerializerNumber(const std::string_view & column_name_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_)
+        ProtobufSerializerNumber(std::string_view column_name_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_)
             : ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_)
         {
             setFunctions();
@@ -590,7 +590,7 @@ namespace
         using ColumnType = std::conditional_t<is_fixed_string, ColumnFixedString, ColumnString>;
 
         ProtobufSerializerString(
-            const std::string_view & column_name_,
+            std::string_view column_name_,
             const std::shared_ptr<const DataTypeFixedString> & fixed_string_data_type_,
             const google::protobuf::FieldDescriptor & field_descriptor_,
             const ProtobufReaderOrWriter & reader_or_writer_)
@@ -604,7 +604,7 @@ namespace
         }
 
         ProtobufSerializerString(
-            const std::string_view & column_name_,
+            std::string_view column_name_,
             const google::protobuf::FieldDescriptor & field_descriptor_,
             const ProtobufReaderOrWriter & reader_or_writer_)
             : ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_)
@@ -727,7 +727,7 @@ namespace
             {
                 case FieldTypeId::TYPE_INT32:
                 {
-                    write_function = [this](const std::string_view & str) { writeInt(parseFromStr<Int32>(str)); };
+                    write_function = [this](std::string_view str) { writeInt(parseFromStr<Int32>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readInt(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); };
                     break;
@@ -735,7 +735,7 @@ namespace
 
                 case FieldTypeId::TYPE_SINT32:
                 {
-                    write_function = [this](const std::string_view & str) { writeSInt(parseFromStr<Int32>(str)); };
+                    write_function = [this](std::string_view str) { writeSInt(parseFromStr<Int32>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readSInt(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); };
                     break;
@@ -743,7 +743,7 @@ namespace
 
                 case FieldTypeId::TYPE_UINT32:
                 {
-                    write_function = [this](const std::string_view & str) { writeUInt(parseFromStr<UInt32>(str)); };
+                    write_function = [this](std::string_view str) { writeUInt(parseFromStr<UInt32>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readUInt(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_uint32()); };
                     break;
@@ -751,7 +751,7 @@ namespace
 
                 case FieldTypeId::TYPE_INT64:
                 {
-                    write_function = [this](const std::string_view & str) { writeInt(parseFromStr<Int64>(str)); };
+                    write_function = [this](std::string_view str) { writeInt(parseFromStr<Int64>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readInt(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); };
                     break;
@@ -759,7 +759,7 @@ namespace
 
                 case FieldTypeId::TYPE_SINT64:
                 {
-                    write_function = [this](const std::string_view & str) { writeSInt(parseFromStr<Int64>(str)); };
+                    write_function = [this](std::string_view str) { writeSInt(parseFromStr<Int64>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readSInt(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); };
                     break;
@@ -767,7 +767,7 @@ namespace
 
                 case FieldTypeId::TYPE_UINT64:
                 {
-                    write_function = [this](const std::string_view & str) { writeUInt(parseFromStr<UInt64>(str)); };
+                    write_function = [this](std::string_view str) { writeUInt(parseFromStr<UInt64>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readUInt(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_uint64()); };
                     break;
@@ -775,7 +775,7 @@ namespace
 
                 case FieldTypeId::TYPE_FIXED32:
                 {
-                    write_function = [this](const std::string_view & str) { writeFixed<UInt32>(parseFromStr<UInt32>(str)); };
+                    write_function = [this](std::string_view str) { writeFixed<UInt32>(parseFromStr<UInt32>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<UInt32>(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_uint32()); };
                     break;
@@ -783,7 +783,7 @@ namespace
 
                 case FieldTypeId::TYPE_SFIXED32:
                 {
-                    write_function = [this](const std::string_view & str) { writeFixed<Int32>(parseFromStr<Int32>(str)); };
+                    write_function = [this](std::string_view str) { writeFixed<Int32>(parseFromStr<Int32>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Int32>(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); };
                     break;
@@ -791,7 +791,7 @@ namespace
 
                 case FieldTypeId::TYPE_FIXED64:
                 {
-                    write_function = [this](const std::string_view & str) { writeFixed<UInt64>(parseFromStr<UInt64>(str)); };
+                    write_function = [this](std::string_view str) { writeFixed<UInt64>(parseFromStr<UInt64>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<UInt64>(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_uint64()); };
                     break;
@@ -799,7 +799,7 @@ namespace
 
                 case FieldTypeId::TYPE_SFIXED64:
                 {
-                    write_function = [this](const std::string_view & str) { writeFixed<Int64>(parseFromStr<Int64>(str)); };
+                    write_function = [this](std::string_view str) { writeFixed<Int64>(parseFromStr<Int64>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Int64>(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); };
                     break;
@@ -807,7 +807,7 @@ namespace
 
                 case FieldTypeId::TYPE_FLOAT:
                 {
-                    write_function = [this](const std::string_view & str) { writeFixed<Float32>(parseFromStr<Float32>(str)); };
+                    write_function = [this](std::string_view str) { writeFixed<Float32>(parseFromStr<Float32>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Float32>(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_float()); };
                     break;
@@ -815,7 +815,7 @@ namespace
 
                 case FieldTypeId::TYPE_DOUBLE:
                 {
-                    write_function = [this](const std::string_view & str) { writeFixed<Float64>(parseFromStr<Float64>(str)); };
+                    write_function = [this](std::string_view str) { writeFixed<Float64>(parseFromStr<Float64>(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Float64>(), str); };
                     default_function = [this]() -> String { return toString(field_descriptor.default_value_double()); };
                     break;
@@ -823,7 +823,7 @@ namespace
 
                 case FieldTypeId::TYPE_BOOL:
                 {
-                    write_function = [this](const std::string_view & str)
+                    write_function = [this](std::string_view str)
                     {
                         if (str == "true")
                             writeUInt(1);
@@ -855,7 +855,7 @@ namespace
                 case FieldTypeId::TYPE_STRING:
                 case FieldTypeId::TYPE_BYTES:
                 {
-                    write_function = [this](const std::string_view & str) { writeStr(str); };
+                    write_function = [this](std::string_view str) { writeStr(str); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { readStrAndAppend(str); };
                     default_function = [this]() -> String { return field_descriptor.default_value_string(); };
                     break;
@@ -863,7 +863,7 @@ namespace
 
                 case FieldTypeId::TYPE_ENUM:
                 {
-                    write_function = [this](const std::string_view & str) { writeInt(stringToProtobufEnumValue(str)); };
+                    write_function = [this](std::string_view str) { writeInt(stringToProtobufEnumValue(str)); };
                     read_function = [this](PaddedPODArray<UInt8> & str) { protobufEnumValueToStringAppend(readInt(), str); };
                     default_function = [this]() -> String { return field_descriptor.default_value_enum()->name(); };
                     break;
@@ -908,7 +908,7 @@ namespace
             }
         }
 
-        int stringToProtobufEnumValue(const std::string_view & str) const
+        int stringToProtobufEnumValue(std::string_view str) const
         {
             auto it = string_to_protobuf_enum_value_map.find(str);
             if (it == string_to_protobuf_enum_value_map.end())
@@ -932,7 +932,7 @@ namespace
 
         const std::shared_ptr<const DataTypeFixedString> fixed_string_data_type;
         const size_t n = 0;
-        std::function<void(const std::string_view &)> write_function;
+        std::function<void(std::string_view)> write_function;
         std::function<void(PaddedPODArray<UInt8> &)> read_function;
         std::function<String()> default_function;
         std::unordered_map<std::string_view, int> string_to_protobuf_enum_value_map;
@@ -953,7 +953,7 @@ namespace
         using BaseClass = ProtobufSerializerNumber<NumberType>;
 
         ProtobufSerializerEnum(
-            const std::string_view & column_name_,
+            std::string_view column_name_,
             const std::shared_ptr<const EnumDataType> & enum_data_type_,
             const FieldDescriptor & field_descriptor_,
             const ProtobufReaderOrWriter & reader_or_writer_)
@@ -1067,7 +1067,7 @@ namespace
                     protobuf_enum_value_to_enum_data_type_value_map.emplace(protobuf_enum_value, enum_data_type_value);
             };
 
-            auto iless = [](const std::string_view & s1, const std::string_view & s2) { return ColumnNameWithProtobufFieldNameComparator::less(s1, s2); };
+            auto iless = [](std::string_view s1, std::string_view s2) { return ColumnNameWithProtobufFieldNameComparator::less(s1, s2); };
             boost::container::flat_map<std::string_view, int, decltype(iless)> string_to_protobuf_enum_value_map;
             typename decltype(string_to_protobuf_enum_value_map)::sequence_type string_to_protobuf_enum_value_seq;
             for (int i : collections::range(enum_descriptor.value_count()))
@@ -1133,9 +1133,9 @@ namespace
 
         Int64 readInt() { return ProtobufSerializerSingleValue::readInt(); }
         void writeInt(Int64 value) { ProtobufSerializerSingleValue::writeInt(value); }
-        void writeStr(const std::string_view & str) { ProtobufSerializerSingleValue::writeStr(str); }
+        void writeStr(std::string_view str) { ProtobufSerializerSingleValue::writeStr(str); }
         void readStr(String & str) { ProtobufSerializerSingleValue::readStr(str); }
-        [[noreturn]] void cannotConvertValue(const std::string_view & src_value, const std::string_view & src_type_name, const std::string_view & dest_type_name) const { ProtobufSerializerSingleValue::cannotConvertValue(src_value, src_type_name, dest_type_name); }
+        [[noreturn]] void cannotConvertValue(std::string_view src_value, std::string_view src_type_name, std::string_view dest_type_name) const { ProtobufSerializerSingleValue::cannotConvertValue(src_value, src_type_name, dest_type_name); }
 
         const std::shared_ptr<const EnumDataType> enum_data_type;
         std::unordered_map<NumberType, int> enum_data_type_value_to_protobuf_enum_value_map;
@@ -1152,7 +1152,7 @@ namespace
         using ColumnType = ColumnDecimal<DecimalType>;
 
         ProtobufSerializerDecimal(
-            const std::string_view & column_name_,
+            std::string_view column_name_,
             const DataTypeDecimalBase<DecimalType> & decimal_data_type_,
             const FieldDescriptor & field_descriptor_,
             const ProtobufReaderOrWriter & reader_or_writer_)
@@ -1412,7 +1412,7 @@ namespace
     {
     public:
         ProtobufSerializerDate(
-            const std::string_view & column_name_,
+            std::string_view column_name_,
             const FieldDescriptor & field_descriptor_,
             const ProtobufReaderOrWriter & reader_or_writer_)
             : ProtobufSerializerNumber<UInt16>(column_name_, field_descriptor_, reader_or_writer_)
@@ -1490,7 +1490,7 @@ namespace
     {
     public:
         ProtobufSerializerDateTime(
-            const std::string_view & column_name_,
+            std::string_view column_name_,
             const DataTypeDateTime & type,
             const FieldDescriptor & field_descriptor_,
             const ProtobufReaderOrWriter & reader_or_writer_)
@@ -1574,7 +1574,7 @@ namespace
     {
     public:
         ProtobufSerializerUUID(
-            const std::string_view & column_name_,
+            std::string_view column_name_,
             const google::protobuf::FieldDescriptor & field_descriptor_,
             const ProtobufReaderOrWriter & reader_or_writer_)
             : ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_)
@@ -1654,7 +1654,7 @@ namespace
     {
     public:
         ProtobufSerializerAggregateFunction(
-            const std::string_view & column_name_,
+            std::string_view column_name_,
             const std::shared_ptr<const DataTypeAggregateFunction> & aggregate_function_data_type_,
             const google::protobuf::FieldDescriptor & field_descriptor_,
             const ProtobufReaderOrWriter & reader_or_writer_)
@@ -2061,7 +2061,7 @@ namespace
     {
     public:
         ProtobufSerializerTupleAsArray(
-            const std::string_view & column_name_,
+            std::string_view column_name_,
             const std::shared_ptr<const DataTypeTuple> & tuple_data_type_,
             const FieldDescriptor & field_descriptor_,
             std::vector<std::unique_ptr<ProtobufSerializer>> element_serializers_)
@@ -2833,7 +2833,7 @@ namespace
             return field_names;
         }
 
-        static bool columnNameEqualsToFieldName(const std::string_view & column_name, const FieldDescriptor & field_descriptor)
+        static bool columnNameEqualsToFieldName(std::string_view column_name, const FieldDescriptor & field_descriptor)
         {
             std::string_view suffix;
             return columnNameStartsWithFieldName(column_name, field_descriptor, suffix) && suffix.empty();
@@ -2844,7 +2844,7 @@ namespace
         /// which doesn't match to the field's name.
         /// The function requires that rest part of the column's name to be started with a dot '.' or underline '_',
         /// but doesn't include those '.' or '_' characters into `suffix`.
-        static bool columnNameStartsWithFieldName(const std::string_view & column_name, const FieldDescriptor & field_descriptor, std::string_view & suffix)
+        static bool columnNameStartsWithFieldName(std::string_view column_name, const FieldDescriptor & field_descriptor, std::string_view & suffix)
         {
             size_t matching_length = 0;
             const MessageDescriptor & containing_type = *field_descriptor.containing_type();
@@ -2887,7 +2887,7 @@ namespace
         /// for that case suffixes are also returned.
         /// This is only the first filter, buildMessageSerializerImpl() does other checks after calling this function.
         static bool findFieldsByColumnName(
-            const std::string_view & column_name,
+            std::string_view column_name,
             const MessageDescriptor & message_descriptor,
             std::vector<std::pair<const FieldDescriptor *, std::string_view /* suffix */>> & out_field_descriptors_with_suffixes,
             bool google_wrappers_special_treatment)
@@ -3030,7 +3030,7 @@ namespace
             used_column_indices_sorted.reserve(num_columns);
             size_t sequential_column_index = 0;
 
-            auto add_field_serializer = [&](const std::string_view & column_name_,
+            auto add_field_serializer = [&](std::string_view column_name_,
                                             std::vector<size_t> && column_indices_,
                                             const FieldDescriptor & field_descriptor_,
                                             std::unique_ptr<ProtobufSerializer> field_serializer_)
@@ -3243,7 +3243,7 @@ namespace
         /// Builds a serializer for one-to-one match:
         /// one column is serialized as one field in the protobuf message.
         std::unique_ptr<ProtobufSerializer> buildFieldSerializer(
-            const std::string_view & column_name,
+            std::string_view column_name,
             const DataTypePtr & data_type,
             const FieldDescriptor & field_descriptor,
             bool allow_repeat,
@@ -3395,7 +3395,7 @@ namespace
             }
         }
 
-        [[noreturn]] static void throwFieldNotRepeated(const FieldDescriptor & field_descriptor, const std::string_view & column_name)
+        [[noreturn]] static void throwFieldNotRepeated(const FieldDescriptor & field_descriptor, std::string_view column_name)
         {
             if (!field_descriptor.is_repeated())
                 throw Exception(

src/Formats/ProtobufWriter.cpp

@@ -196,7 +196,7 @@ template void ProtobufWriter::writeFixed<UInt64>(int field_number, UInt64 value)
 template void ProtobufWriter::writeFixed<Float32>(int field_number, Float32 value);
 template void ProtobufWriter::writeFixed<Float64>(int field_number, Float64 value);
 
-void ProtobufWriter::writeString(int field_number, const std::string_view & str)
+void ProtobufWriter::writeString(int field_number, std::string_view str)
 {
     size_t length = str.length();
     size_t old_size = buffer.size();

src/Formats/ProtobufWriter.h

@@ -30,7 +30,7 @@ public:
     void writeSInt(int field_number, Int64 value);
     template <typename T>
     void writeFixed(int field_number, T value);
-    void writeString(int field_number, const std::string_view & str);
+    void writeString(int field_number, std::string_view str);
 
     void startRepeatedPack();
     void endRepeatedPack(int field_number, bool skip_if_empty);

src/Functions/FunctionsAES.cpp

@@ -25,7 +25,7 @@ void onError(std::string error_message)
     throw DB::Exception(error_message, DB::ErrorCodes::OPENSSL_ERROR);
 }
 
-StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array<char, EVP_MAX_KEY_LENGTH> & folded_key)
+StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, StringRef key, std::array<char, EVP_MAX_KEY_LENGTH> & folded_key)
 {
     assert(cipher_key_size <= EVP_MAX_KEY_LENGTH);
     memcpy(folded_key.data(), key.data, cipher_key_size);
@@ -38,7 +38,7 @@ StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const
     return StringRef(folded_key.data(), cipher_key_size);
 }
 
-const EVP_CIPHER * getCipherByName(const StringRef & cipher_name)
+const EVP_CIPHER * getCipherByName(StringRef cipher_name)
 {
     // NOTE: cipher obtained not via EVP_CIPHER_fetch() would cause extra work on each context reset
     // with EVP_CIPHER_CTX_reset() or EVP_EncryptInit_ex(), but using EVP_CIPHER_fetch()

src/Functions/FunctionsAES.h

@@ -32,9 +32,9 @@ namespace ErrorCodes
 namespace OpenSSLDetails
 {
 [[noreturn]] void onError(std::string error_message);
-StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array<char, EVP_MAX_KEY_LENGTH> & folded_key);
+StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, StringRef key, std::array<char, EVP_MAX_KEY_LENGTH> & folded_key);
 
-const EVP_CIPHER * getCipherByName(const StringRef & name);
+const EVP_CIPHER * getCipherByName(StringRef name);
 
 enum class CompatibilityMode
 {
@@ -53,7 +53,7 @@ enum class CipherMode
 template <CipherMode mode>
 struct KeyHolder
 {
-    inline StringRef setKey(size_t cipher_key_size, const StringRef & key) const
+    inline StringRef setKey(size_t cipher_key_size, StringRef key) const
     {
         if (key.size != cipher_key_size)
             throw DB::Exception(fmt::format("Invalid key size: {} expected {}", key.size, cipher_key_size),
@@ -66,7 +66,7 @@ struct KeyHolder
 template <>
 struct KeyHolder<CipherMode::MySQLCompatibility>
 {
-    inline StringRef setKey(size_t cipher_key_size, const StringRef & key)
+    inline StringRef setKey(size_t cipher_key_size, StringRef key)
     {
         if (key.size < cipher_key_size)
             throw DB::Exception(fmt::format("Invalid key size: {} expected {}", key.size, cipher_key_size),
@@ -120,7 +120,7 @@ inline void validateCipherMode(const EVP_CIPHER * evp_cipher)
 }
 
 template <CipherMode mode>
-inline void validateIV(const StringRef & iv_value, const size_t cipher_iv_size)
+inline void validateIV(StringRef iv_value, const size_t cipher_iv_size)
 {
     // In MySQL mode we don't care if IV is longer than expected, only if shorter.
     if ((mode == CipherMode::MySQLCompatibility && iv_value.size != 0 && iv_value.size < cipher_iv_size)

src/Functions/FunctionsJSON.cpp

@@ -304,7 +304,7 @@ private:
 
     /// Performs moves of types MoveType::Key and MoveType::ConstKey.
     template <typename JSONParser>
-    static bool moveToElementByKey(typename JSONParser::Element & element, const std::string_view & key)
+    static bool moveToElementByKey(typename JSONParser::Element & element, std::string_view key)
     {
         if (!element.isObject())
             return false;
@@ -504,7 +504,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view &)
+    static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view)
     {
         ColumnVector<UInt8> & col_vec = assert_cast<ColumnVector<UInt8> &>(dest);
         col_vec.insertValue(1);
@@ -532,7 +532,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName &) { return 0; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view &)
+    static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view)
     {
         /// This function is called only if JSON is valid.
         /// If JSON isn't valid then `FunctionJSON::Executor::run()` adds default value (=zero) to `dest` without calling this function.
@@ -556,7 +556,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         size_t size;
         if (element.isArray())
@@ -586,7 +586,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view & last_key)
+    static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view last_key)
     {
         if (last_key.empty())
             return false;
@@ -620,7 +620,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         UInt8 type;
         if (element.isInt64())
@@ -662,7 +662,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         NumberType value;
 
@@ -737,7 +737,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         if (!element.isBool())
             return false;
@@ -764,7 +764,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         if (element.isNull())
             return false;
@@ -1164,7 +1164,7 @@ public:
         extract_tree = JSONExtractTree<JSONParser>::build(function_name, result_type);
     }
 
-    bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         return extract_tree->insertResultToColumn(dest, element);
     }
@@ -1207,7 +1207,7 @@ public:
         extract_tree = JSONExtractTree<JSONParser>::build(function_name, value_type);
     }
 
-    bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         if (!element.isObject())
             return false;
@@ -1251,7 +1251,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         ColumnString & col_str = assert_cast<ColumnString &>(dest);
         auto & chars = col_str.getChars();
@@ -1355,7 +1355,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         if (!element.isArray())
             return false;
@@ -1387,7 +1387,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         if (!element.isObject())
             return false;
@@ -1423,7 +1423,7 @@ public:
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
+    bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
     {
         if (!element.isObject())
             return false;

src/Functions/isIPAddressContainedIn.cpp

@@ -27,7 +27,7 @@ class IPAddressVariant
 {
 public:
 
-    explicit IPAddressVariant(const StringRef & address_str)
+    explicit IPAddressVariant(StringRef address_str)
     {
         /// IP address parser functions require that the input is
         /// NULL-terminated so we need to copy it.
@@ -75,7 +75,7 @@ struct IPAddressCIDR
     UInt8 prefix;
 };
 
-IPAddressCIDR parseIPWithCIDR(const StringRef cidr_str)
+IPAddressCIDR parseIPWithCIDR(StringRef cidr_str)
 {
     std::string_view cidr_str_view(cidr_str);
     size_t pos_slash = cidr_str_view.find('/');

src/Functions/map.cpp

@@ -562,15 +562,23 @@ public:
     {
         const ColumnMap * col_map_left = typeid_cast<const ColumnMap *>(arguments[0].column.get());
         const auto * col_const_map_left = checkAndGetColumnConst<ColumnMap>(arguments[0].column.get());
+        bool col_const_map_left_flag = false;
         if (col_const_map_left)
+        {
+            col_const_map_left_flag = true;
             col_map_left = typeid_cast<const ColumnMap *>(&col_const_map_left->getDataColumn());
+        }
         if (!col_map_left)
             return nullptr;
 
         const ColumnMap * col_map_right = typeid_cast<const ColumnMap *>(arguments[1].column.get());
         const auto * col_const_map_right = checkAndGetColumnConst<ColumnMap>(arguments[1].column.get());
+        bool col_const_map_right_flag = false;
         if (col_const_map_right)
+        {
+            col_const_map_right_flag = true;
             col_map_right = typeid_cast<const ColumnMap *>(&col_const_map_right->getDataColumn());
+        }
         if (!col_map_right)
             return nullptr;
 
@@ -592,13 +600,18 @@ public:
         MutableColumnPtr offsets = DataTypeNumber<IColumn::Offset>().createColumn();
 
         IColumn::Offset current_offset = 0;
-        for (size_t idx = 0; idx < input_rows_count; ++idx)
+        for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
         {
-            for (size_t i = offsets_left[idx - 1]; i < offsets_left[idx]; ++i)
+            size_t left_it_begin = col_const_map_left_flag ? 0 : offsets_left[row_idx - 1];
+            size_t left_it_end = col_const_map_left_flag ? offsets_left.size() : offsets_left[row_idx];
+            size_t right_it_begin = col_const_map_right_flag ? 0 : offsets_right[row_idx - 1];
+            size_t right_it_end = col_const_map_right_flag ? offsets_right.size() : offsets_right[row_idx];
+
+            for (size_t i = left_it_begin; i < left_it_end; ++i)
             {
                 bool matched = false;
                 auto key = keys_data_left.getDataAt(i);
-                for (size_t j = offsets_right[idx - 1]; j < offsets_right[idx]; ++j)
+                for (size_t j = right_it_begin; j < right_it_end; ++j)
                 {
                     if (keys_data_right.getDataAt(j).toString() == key.toString())
                     {
@@ -613,12 +626,14 @@ public:
                     ++current_offset;
                 }
             }
-            for (size_t j = offsets_right[idx - 1]; j < offsets_right[idx]; ++j)
+
+            for (size_t j = right_it_begin; j < right_it_end; ++j)
             {
                 keys_data->insertFrom(keys_data_right, j);
                 values_data->insertFrom(values_data_right, j);
                 ++current_offset;
             }
+
             offsets->insert(current_offset);
         }
 

src/IO/ReadBufferFromAzureBlobStorage.cpp

@@ -5,6 +5,7 @@
 #include <IO/ReadBufferFromAzureBlobStorage.h>
 #include <IO/ReadBufferFromString.h>
 #include <Common/logger_useful.h>
+#include <Common/Throttler.h>
 #include <base/sleep.h>
 
 
@@ -26,7 +27,6 @@ ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage(
     const ReadSettings & read_settings_,
     size_t max_single_read_retries_,
     size_t max_single_download_retries_,
-    size_t tmp_buffer_size_,
     bool use_external_buffer_,
     size_t read_until_position_)
     : ReadBufferFromFileBase(read_settings_.remote_fs_buffer_size, nullptr, 0)
@@ -34,7 +34,8 @@ ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage(
     , path(path_)
     , max_single_read_retries(max_single_read_retries_)
     , max_single_download_retries(max_single_download_retries_)
-    , tmp_buffer_size(tmp_buffer_size_)
+    , read_settings(read_settings_)
+    , tmp_buffer_size(read_settings.remote_fs_buffer_size)
     , use_external_buffer(use_external_buffer_)
     , read_until_position(read_until_position_)
 {
@@ -76,6 +77,8 @@ bool ReadBufferFromAzureBlobStorage::nextImpl()
         try
         {
             bytes_read = data_stream->ReadToCount(reinterpret_cast<uint8_t *>(data_ptr), to_read_bytes);
+            if (read_settings.remote_throttler)
+                read_settings.remote_throttler->add(bytes_read);
             break;
         }
         catch (const Azure::Storage::StorageException & e)

src/IO/ReadBufferFromAzureBlobStorage.h

@@ -23,7 +23,6 @@ public:
         const ReadSettings & read_settings_,
         size_t max_single_read_retries_,
         size_t max_single_download_retries_,
-        size_t tmp_buffer_size_,
         bool use_external_buffer_ = false,
         size_t read_until_position_ = 0);
 
@@ -48,6 +47,7 @@ private:
     const String path;
     size_t max_single_read_retries;
     size_t max_single_download_retries;
+    ReadSettings read_settings;
     std::vector<char> tmp_buffer;
     size_t tmp_buffer_size;
     bool use_external_buffer;

src/IO/ReadBufferFromS3.cpp

@@ -5,12 +5,13 @@
 
 #include <IO/ReadBufferFromIStream.h>
 #include <IO/ReadBufferFromS3.h>
-#include <Common/Stopwatch.h>
 
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/GetObjectRequest.h>
 #include <aws/s3/model/HeadObjectRequest.h>
 
+#include <Common/Stopwatch.h>
+#include <Common/Throttler.h>
 #include <Common/logger_useful.h>
 #include <base/sleep.h>
 
@@ -164,6 +165,8 @@ bool ReadBufferFromS3::nextImpl()
 
     ProfileEvents::increment(ProfileEvents::ReadBufferFromS3Bytes, working_buffer.size());
     offset += working_buffer.size();
+    if (read_settings.remote_throttler)
+        read_settings.remote_throttler->add(working_buffer.size());
 
     return true;
 }
@@ -299,7 +302,6 @@ std::unique_ptr<ReadBuffer> ReadBufferFromS3::initialize()
     if (outcome.IsSuccess())
     {
         read_result = outcome.GetResultWithOwnership();
-
         size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size;
         return std::make_unique<ReadBufferFromIStream>(read_result.GetBody(), buffer_size);
     }

src/IO/ReadHelpers.cpp

@@ -1053,7 +1053,7 @@ template void readDateTimeTextFallback<void>(time_t &, ReadBuffer &, const DateL
 template bool readDateTimeTextFallback<bool>(time_t &, ReadBuffer &, const DateLUTImpl &);
 
 
-void skipJSONField(ReadBuffer & buf, const StringRef & name_of_field)
+void skipJSONField(ReadBuffer & buf, StringRef name_of_field)
 {
     if (buf.eof())
         throw Exception("Unexpected EOF for key '" + name_of_field.toString() + "'", ErrorCodes::INCORRECT_DATA);

src/IO/ReadHelpers.h

@@ -836,7 +836,7 @@ template <typename T>
 inline T parse(const char * data, size_t size);
 
 template <typename T>
-inline T parseFromString(const std::string_view & str)
+inline T parseFromString(std::string_view str)
 {
     return parse<T>(str.data(), str.size());
 }
@@ -1238,7 +1238,7 @@ inline void skipWhitespaceIfAny(ReadBuffer & buf, bool one_line = false)
 }
 
 /// Skips json value.
-void skipJSONField(ReadBuffer & buf, const StringRef & name_of_field);
+void skipJSONField(ReadBuffer & buf, StringRef name_of_field);
 
 
 /** Read serialized exception.
@@ -1338,7 +1338,7 @@ inline T parseWithSizeSuffix(const char * data, size_t size)
 }
 
 template <typename T>
-inline T parseWithSizeSuffix(const std::string_view & s)
+inline T parseWithSizeSuffix(std::string_view s)
 {
     return parseWithSizeSuffix<T>(s.data(), s.size());
 }

src/IO/ReadSettings.h

@@ -4,6 +4,7 @@
 #include <string>
 #include <Core/Defines.h>
 #include <Common/FileCache_fwd.h>
+#include <Common/Throttler_fwd.h>
 
 namespace DB
 {
@@ -89,6 +90,9 @@ struct ReadSettings
 
     FileCachePtr remote_fs_cache;
 
+    /// Bandwidth throttler to use during reading
+    ThrottlerPtr remote_throttler;
+
     size_t http_max_tries = 1;
     size_t http_retry_initial_backoff_ms = 100;
     size_t http_retry_max_backoff_ms = 1600;

src/IO/WriteBufferFromAzureBlobStorage.cpp

@@ -5,6 +5,7 @@
 #include <IO/WriteBufferFromAzureBlobStorage.h>
 #include <Common/getRandomASCIIString.h>
 #include <Common/logger_useful.h>
+#include <Common/Throttler.h>
 
 
 namespace DB
@@ -15,11 +16,13 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage(
     const String & blob_path_,
     size_t max_single_part_upload_size_,
     size_t buf_size_,
+    const WriteSettings & write_settings_,
     std::optional<std::map<std::string, std::string>> attributes_)
     : BufferWithOwnMemory<WriteBuffer>(buf_size_, nullptr, 0)
     , blob_container_client(blob_container_client_)
     , max_single_part_upload_size(max_single_part_upload_size_)
     , blob_path(blob_path_)
+    , write_settings(write_settings_)
     , attributes(attributes_)
 {
 }
@@ -84,6 +87,9 @@ void WriteBufferFromAzureBlobStorage::nextImpl()
     }
 
     block_blob_client.CommitBlockList(block_ids);
+
+    if (write_settings.remote_throttler)
+        write_settings.remote_throttler->add(read);
 }
 
 }