thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-13 18:02:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into skipping_sorting_step

Browse Source
This commit is contained in:
Igor Nikonov 2022-07-15 23:55:39 +02:00 committed by GitHub
commit df9e13d9de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
111 changed files with 860 additions and 589 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/master.yml vendored
View File

@ -2902,7 +2902,7 @@ jobs:
#############################################################################################
#################################### PERFORMANCE TESTS ######################################
#############################################################################################
PerformanceComparison0:
PerformanceComparisonX86-0:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
@ -2940,7 +2940,7 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparison1:
PerformanceComparisonX86-1:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
@ -2978,7 +2978,7 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparison2:
PerformanceComparisonX86-2:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
@ -3016,7 +3016,7 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparison3:
PerformanceComparisonX86-3:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
@ -3096,10 +3096,10 @@ jobs:
- IntegrationTestsTsan1
- IntegrationTestsTsan2
- IntegrationTestsTsan3
- PerformanceComparison0
- PerformanceComparison1
- PerformanceComparison2
- PerformanceComparison3
- PerformanceComparisonX86-0
- PerformanceComparisonX86-1
- PerformanceComparisonX86-2
- PerformanceComparisonX86-3
- CompatibilityCheck
- ASTFuzzerTestDebug
- ASTFuzzerTestAsan

32
.github/workflows/pull_request.yml vendored
View File

@ -3118,7 +3118,7 @@ jobs:
#############################################################################################
#################################### PERFORMANCE TESTS ######################################
#############################################################################################
PerformanceComparison0:
PerformanceComparisonX86-0:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
@ -3156,7 +3156,7 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparison1:
PerformanceComparisonX86-1:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
@ -3194,7 +3194,7 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparison2:
PerformanceComparisonX86-2:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
@ -3232,7 +3232,7 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparison3:
PerformanceComparisonX86-3:
needs: [BuilderDebRelease]
runs-on: [self-hosted, stress-tester]
steps:
@ -3270,7 +3270,7 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonAarch0:
PerformanceComparisonAarch-0:
needs: [BuilderDebAarch64]
runs-on: [self-hosted, func-tester-aarch64]
steps:
@ -3308,7 +3308,7 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonAarch1:
PerformanceComparisonAarch-1:
needs: [BuilderDebAarch64]
runs-on: [self-hosted, func-tester-aarch64]
steps:
@ -3346,7 +3346,7 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonAarch2:
PerformanceComparisonAarch-2:
needs: [BuilderDebAarch64]
runs-on: [self-hosted, func-tester-aarch64]
steps:
@ -3384,7 +3384,7 @@ jobs:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonAarch3:
PerformanceComparisonAarch-3:
needs: [BuilderDebAarch64]
runs-on: [self-hosted, func-tester-aarch64]
steps:
@ -3481,14 +3481,14 @@ jobs:
- IntegrationTestsTsan1
- IntegrationTestsTsan2
- IntegrationTestsTsan3
- PerformanceComparison0
- PerformanceComparison1
- PerformanceComparison2
- PerformanceComparison3
- PerformanceComparisonAarch0
- PerformanceComparisonAarch1
- PerformanceComparisonAarch2
- PerformanceComparisonAarch3
- PerformanceComparisonX86-0
- PerformanceComparisonX86-1
- PerformanceComparisonX86-2
- PerformanceComparisonX86-3
- PerformanceComparisonAarch-0
- PerformanceComparisonAarch-1
- PerformanceComparisonAarch-2
- PerformanceComparisonAarch-3
- UnitTestsAsan
- UnitTestsTsan
- UnitTestsMsan

9
cmake/warnings.cmake
View File

@ -20,13 +20,9 @@ if (COMPILER_CLANG)
# We want to get everything out of the compiler for code quality.
add_warning(everything)
add_warning(pedantic)
no_warning(vla-extension)
no_warning(zero-length-array)
no_warning(c11-extensions)
no_warning(unused-command-line-argument)
no_warning(c++98-compat-pedantic)
no_warning(c++98-compat)
no_warning(c99-extensions)
no_warning(conversion)
no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
no_warning(disabled-macro-expansion)
@ -37,12 +33,7 @@ if (COMPILER_CLANG)
no_warning(global-constructors)
no_warning(missing-prototypes)
no_warning(missing-variable-declarations)
no_warning(nested-anon-types)
no_warning(packed)
no_warning(padded)
no_warning(return-std-move-in-c++11) # clang 7+
no_warning(shift-sign-overflow)
no_warning(sign-conversion)
no_warning(switch-enum)
no_warning(undefined-func-template)
no_warning(unused-template)

2
contrib/thrift vendored

@ -1 +1 @@
Subproject commit 010ccf0a0c7023fea0f6bf4e4078ebdff7e61982
Subproject commit 2a93df80f27739ccabb5b885cb12a8dc7595ecdf

5
contrib/thrift-cmake/CMakeLists.txt
View File

@ -15,7 +15,6 @@ set(thriftcpp_SOURCES
"${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.cpp"
"${LIBRARY_DIR}/src/thrift/concurrency/ThreadManager.cpp"
"${LIBRARY_DIR}/src/thrift/concurrency/TimerManager.cpp"
"${LIBRARY_DIR}/src/thrift/concurrency/Util.cpp"
"${LIBRARY_DIR}/src/thrift/processor/PeekProcessor.cpp"
"${LIBRARY_DIR}/src/thrift/protocol/TBase64Utils.cpp"
"${LIBRARY_DIR}/src/thrift/protocol/TDebugProtocol.cpp"
@ -33,6 +32,8 @@ set(thriftcpp_SOURCES
"${LIBRARY_DIR}/src/thrift/transport/TServerSocket.cpp"
"${LIBRARY_DIR}/src/thrift/transport/TTransportUtils.cpp"
"${LIBRARY_DIR}/src/thrift/transport/TBufferTransports.cpp"
"${LIBRARY_DIR}/src/thrift/transport/SocketCommon.cpp"
"${LIBRARY_DIR}/src/thrift/transport/TWebSocketServer.cpp"
"${LIBRARY_DIR}/src/thrift/server/TConnectedClient.cpp"
"${LIBRARY_DIR}/src/thrift/server/TServerFramework.cpp"
"${LIBRARY_DIR}/src/thrift/server/TSimpleServer.cpp"
@ -92,4 +93,4 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}")
add_library(_thrift ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
add_library(ch_contrib::thrift ALIAS _thrift)
target_include_directories(_thrift SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src" ${CMAKE_CURRENT_BINARY_DIR})
target_link_libraries (_thrift PUBLIC boost::headers_only)
target_link_libraries (_thrift PUBLIC OpenSSL::SSL boost::headers_only)

1
docker/test/fasttest/Dockerfile
View File

@ -71,6 +71,7 @@ RUN apt-get update \
software-properties-common \
tzdata \
unixodbc \
file \
--yes --no-install-recommends
RUN pip3 install numpy scipy pandas Jinja2

1
docker/test/stateless/Dockerfile
View File

@ -34,6 +34,7 @@ RUN apt-get update -y \
unixodbc \
wget \
zstd \
file \
&& apt-get clean

7
docker/test/stress/run.sh
View File

@ -322,7 +322,11 @@ else
clickhouse-client --query="SELECT 'Server version: ', version()"
# Install new package before running stress test because we should use new clickhouse-client and new clickhouse-test
# But we should leave old binary in /usr/bin/ for gdb (so it will print sane stacktarces)
mv /usr/bin/clickhouse previous_release_package_folder/
install_packages package_folder
mv /usr/bin/clickhouse package_folder/
mv previous_release_package_folder/clickhouse /usr/bin/
mkdir tmp_stress_output
@ -337,6 +341,7 @@ else
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
# Start new server
mv package_folder/clickhouse /usr/bin/
configure
start 500
clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
@ -356,6 +361,7 @@ else
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64")
# FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
# Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
echo "Check for Error messages in server log:"
zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
-e "Code: 236. DB::Exception: Cancelled mutating parts" \
@ -382,6 +388,7 @@ else
-e "Cannot parse string 'Hello' as UInt64" \
-e "} <Error> TCPHandler: Code:" \
-e "} <Error> executeQuery: Code:" \
-e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
/var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|| echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv

2
programs/copier/ZooKeeperStaff.h
View File

@ -20,7 +20,7 @@ public:
bool operator<=(const WrappingUInt32 & other) const
{
const UInt32 HALF = 1 << 31;
const UInt32 HALF = static_cast<UInt32>(1) << 31;
return (value <= other.value && other.value - value < HALF)
|| (value > other.value && value - other.value > HALF);
}

4
src/Access/AccessControl.cpp
View File

@ -101,7 +101,7 @@ public:
registered_prefixes = prefixes_;
}
bool isSettingNameAllowed(const std::string_view & setting_name) const
bool isSettingNameAllowed(std::string_view setting_name) const
{
if (Settings::hasBuiltin(setting_name))
return true;
@ -116,7 +116,7 @@ public:
return false;
}
void checkSettingNameIsAllowed(const std::string_view & setting_name) const
void checkSettingNameIsAllowed(std::string_view setting_name) const
{
if (isSettingNameAllowed(setting_name))
return;

70
src/Access/AccessRights.cpp
View File

@ -252,7 +252,7 @@ public:
}
template <typename ... Args>
void grant(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
void grant(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
{
auto & child = getChild(name);
child.grant(flags_, subnames...);
@ -279,7 +279,7 @@ public:
}
template <typename... Args>
void revoke(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
void revoke(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
{
auto & child = getChild(name);
@ -306,7 +306,7 @@ public:
}
template <typename... Args>
bool isGranted(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames) const
bool isGranted(const AccessFlags & flags_, std::string_view name, const Args &... subnames) const
{
AccessFlags flags_to_check = flags_ - min_flags_with_children;
if (!flags_to_check)
@ -415,7 +415,7 @@ private:
AccessFlags getAllGrantableFlags() const { return ::DB::getAllGrantableFlags(level); }
AccessFlags getChildAllGrantableFlags() const { return ::DB::getAllGrantableFlags(static_cast<Level>(level + 1)); }
Node * tryGetChild(const std::string_view & name) const
Node * tryGetChild(std::string_view name) const
{
if (!children)
return nullptr;
@ -425,7 +425,7 @@ private:
return &it->second;
}
Node & getChild(const std::string_view & name)
Node & getChild(std::string_view name)
{
auto * child = tryGetChild(name);
if (child)
@ -819,20 +819,20 @@ void AccessRights::grantImpl(const AccessRightsElements & elements)
}
void AccessRights::grant(const AccessFlags & flags) { grantImpl<false>(flags); }
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl<false>(flags, database); }
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<false>(flags, database, table); }
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<false>(flags, database, table, column); }
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
void AccessRights::grant(const AccessFlags & flags, std::string_view database) { grantImpl<false>(flags, database); }
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<false>(flags, database, table); }
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<false>(flags, database, table, column); }
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
void AccessRights::grant(const AccessRightsElement & element) { grantImpl<false>(element); }
void AccessRights::grant(const AccessRightsElements & elements) { grantImpl<false>(elements); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl<true>(flags); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database) { grantImpl<true>(flags, database); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<true>(flags, database, table); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<true>(flags, database, table, column); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database) { grantImpl<true>(flags, database); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<true>(flags, database, table); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<true>(flags, database, table, column); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
void AccessRights::grantWithGrantOption(const AccessRightsElement & element) { grantImpl<true>(element); }
void AccessRights::grantWithGrantOption(const AccessRightsElements & elements) { grantImpl<true>(elements); }
@ -892,20 +892,20 @@ void AccessRights::revokeImpl(const AccessRightsElements & elements)
}
void AccessRights::revoke(const AccessFlags & flags) { revokeImpl<false>(flags); }
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl<false>(flags, database); }
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<false>(flags, database, table); }
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<false>(flags, database, table, column); }
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
void AccessRights::revoke(const AccessFlags & flags, std::string_view database) { revokeImpl<false>(flags, database); }
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<false>(flags, database, table); }
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<false>(flags, database, table, column); }
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
void AccessRights::revoke(const AccessRightsElement & element) { revokeImpl<false>(element); }
void AccessRights::revoke(const AccessRightsElements & elements) { revokeImpl<false>(elements); }
void AccessRights::revokeGrantOption(const AccessFlags & flags) { revokeImpl<true>(flags); }
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database) { revokeImpl<true>(flags, database); }
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<true>(flags, database, table); }
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<true>(flags, database, table, column); }
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database) { revokeImpl<true>(flags, database); }
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<true>(flags, database, table); }
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<true>(flags, database, table, column); }
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
void AccessRights::revokeGrantOption(const AccessRightsElement & element) { revokeImpl<true>(element); }
void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { revokeImpl<true>(elements); }
@ -984,20 +984,20 @@ bool AccessRights::isGrantedImpl(const AccessRightsElements & elements) const
}
bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl<false>(flags); }
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<false>(flags, database); }
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<false>(flags, database, table); }
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<false>(flags, database, table, column); }
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<false>(flags, database); }
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<false>(flags, database, table); }
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<false>(flags, database, table, column); }
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
bool AccessRights::isGranted(const AccessRightsElement & element) const { return isGrantedImpl<false>(element); }
bool AccessRights::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl<false>(elements); }
bool AccessRights::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl<true>(flags); }
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<true>(flags, database); }
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<true>(flags, database, table); }
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<true>(flags, database, table, column); }
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<true>(flags, database); }
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<true>(flags, database, table); }
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<true>(flags, database, table, column); }
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl<true>(element); }
bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl<true>(elements); }

66
src/Access/AccessRights.h
View File

@ -39,59 +39,59 @@ public:
/// Grants access on a specified database/table/column.
/// Does nothing if the specified access has been already granted.
void grant(const AccessFlags & flags);
void grant(const AccessFlags & flags, const std::string_view & database);
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
void grant(const AccessFlags & flags, std::string_view database);
void grant(const AccessFlags & flags, std::string_view database, std::string_view table);
void grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
void grant(const AccessRightsElement & element);
void grant(const AccessRightsElements & elements);
void grantWithGrantOption(const AccessFlags & flags);
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database);
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
void grantWithGrantOption(const AccessFlags & flags, std::string_view database);
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
void grantWithGrantOption(const AccessRightsElement & element);
void grantWithGrantOption(const AccessRightsElements & elements);
/// Revokes a specified access granted earlier on a specified database/table/column.
/// For example, revoke(AccessType::ALL) revokes all grants at all, just like clear();
void revoke(const AccessFlags & flags);
void revoke(const AccessFlags & flags, const std::string_view & database);
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
void revoke(const AccessFlags & flags, std::string_view database);
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table);
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
void revoke(const AccessRightsElement & element);
void revoke(const AccessRightsElements & elements);
void revokeGrantOption(const AccessFlags & flags);
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database);
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
void revokeGrantOption(const AccessFlags & flags, std::string_view database);
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
void revokeGrantOption(const AccessRightsElement & element);
void revokeGrantOption(const AccessRightsElements & elements);
/// Whether a specified access granted.
bool isGranted(const AccessFlags & flags) const;
bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
bool isGranted(const AccessFlags & flags, std::string_view database) const;
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
bool isGranted(const AccessRightsElement & element) const;
bool isGranted(const AccessRightsElements & elements) const;
bool hasGrantOption(const AccessFlags & flags) const;
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
bool hasGrantOption(const AccessRightsElement & element) const;
bool hasGrantOption(const AccessRightsElements & elements) const;
@ -107,9 +107,9 @@ public:
const AccessFlags & flags,
const AccessFlags & min_flags_with_children,
const AccessFlags & max_flags_with_children,
const std::string_view & database,
const std::string_view & table,
const std::string_view & column)>;
std::string_view database,
std::string_view table,
std::string_view column)>;
void modifyFlags(const ModifyFlagsFunction & function);
void modifyFlagsWithGrantOption(const ModifyFlagsFunction & function);

8
src/Access/Authentication.cpp
View File

@ -26,17 +26,17 @@ namespace
return (Util::stringToDigest(password) == password_plaintext);
}
bool checkPasswordDoubleSHA1(const std::string_view & password, const Digest & password_double_sha1)
bool checkPasswordDoubleSHA1(std::string_view password, const Digest & password_double_sha1)
{
return (Util::encodeDoubleSHA1(password) == password_double_sha1);
}
bool checkPasswordSHA256(const std::string_view & password, const Digest & password_sha256, const String & salt)
bool checkPasswordSHA256(std::string_view password, const Digest & password_sha256, const String & salt)
{
return Util::encodeSHA256(String(password).append(salt)) == password_sha256;
}
bool checkPasswordDoubleSHA1MySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_double_sha1)
bool checkPasswordDoubleSHA1MySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_double_sha1)
{
/// scrambled_password = SHA1(password) XOR SHA1(scramble <concat> SHA1(SHA1(password)))
@ -61,7 +61,7 @@ namespace
return calculated_password_double_sha1 == password_double_sha1;
}
bool checkPasswordPlainTextMySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_plaintext)
bool checkPasswordPlainTextMySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_plaintext)
{
return checkPasswordDoubleSHA1MySQL(scramble, scrambled_password, Util::encodeDoubleSHA1(password_plaintext));
}

14
src/Access/Common/AccessFlags.cpp
View File

@ -35,7 +35,7 @@ namespace
return access_type_to_flags_mapping[static_cast<size_t>(type)];
}
Flags keywordToFlags(const std::string_view & keyword) const
Flags keywordToFlags(std::string_view keyword) const
{
auto it = keyword_to_flags_map.find(keyword);
if (it == keyword_to_flags_map.end())
@ -142,14 +142,14 @@ namespace
}
};
static String replaceUnderscoreWithSpace(const std::string_view & str)
static String replaceUnderscoreWithSpace(std::string_view str)
{
String res{str};
boost::replace_all(res, "_", " ");
return res;
}
static Strings splitAliases(const std::string_view & str)
static Strings splitAliases(std::string_view str)
{
Strings aliases;
boost::split(aliases, str, boost::is_any_of(","));
@ -160,10 +160,10 @@ namespace
static void makeNode(
AccessType access_type,
const std::string_view & name,
const std::string_view & aliases,
std::string_view name,
std::string_view aliases,
NodeType node_type,
const std::string_view & parent_group_name,
std::string_view parent_group_name,
std::unordered_map<std::string_view, Node *> & nodes,
std::unordered_map<std::string_view, NodePtr> & owned_nodes,
size_t & next_flag)
@ -353,7 +353,7 @@ namespace
AccessFlags::AccessFlags(AccessType type) : flags(Helper::instance().accessTypeToFlags(type)) {}
AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
AccessFlags::AccessFlags(std::string_view keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
AccessFlags::AccessFlags(const std::vector<std::string_view> & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
AccessFlags::AccessFlags(const Strings & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
String AccessFlags::toString() const { return Helper::instance().flagsToString(flags); }

2
src/Access/Common/AccessFlags.h
View File

@ -21,7 +21,7 @@ public:
AccessFlags() = default;
/// Constructs from a string like "SELECT".
AccessFlags(const std::string_view & keyword); /// NOLINT
AccessFlags(std::string_view keyword); /// NOLINT
/// Constructs from a list of strings like "SELECT, UPDATE, INSERT".
AccessFlags(const std::vector<std::string_view> & keywords); /// NOLINT

14
src/Access/Common/AccessRightsElement.cpp
View File

@ -81,7 +81,7 @@ namespace
}
bool need_comma = false;
for (const std::string_view & keyword : keywords)
for (std::string_view keyword : keywords)
{
if (need_comma)
result.append(", ");
@ -145,18 +145,18 @@ namespace
}
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_)
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_)
: access_flags(access_flags_), database(database_), any_database(false)
{
}
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_)
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_)
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false)
{
}
AccessRightsElement::AccessRightsElement(
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_)
AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_)
: access_flags(access_flags_)
, database(database_)
, table(table_)
@ -169,8 +169,8 @@ AccessRightsElement::AccessRightsElement(
AccessRightsElement::AccessRightsElement(
AccessFlags access_flags_,
const std::string_view & database_,
const std::string_view & table_,
std::string_view database_,
std::string_view table_,
const std::vector<std::string_view> & columns_)
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false), any_column(false)
{
@ -180,7 +180,7 @@ AccessRightsElement::AccessRightsElement(
}
AccessRightsElement::AccessRightsElement(
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_)
AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_)
: access_flags(access_flags_)
, database(database_)
, table(table_)

12
src/Access/Common/AccessRightsElement.h
View File

@ -28,19 +28,19 @@ struct AccessRightsElement
explicit AccessRightsElement(AccessFlags access_flags_) : access_flags(access_flags_) {}
AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_);
AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_);
AccessRightsElement(AccessFlags access_flags_, std::string_view database_);
AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_);
AccessRightsElement(
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_);
AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_);
AccessRightsElement(
AccessFlags access_flags_,
const std::string_view & database_,
const std::string_view & table_,
std::string_view database_,
std::string_view table_,
const std::vector<std::string_view> & columns_);
AccessRightsElement(
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_);
AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_);
bool empty() const { return !access_flags || (!any_column && columns.empty()); }

2
src/Access/Common/AccessType.cpp
View File

@ -35,7 +35,7 @@ namespace
#undef ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING
}
void addToMapping(AccessType type, const std::string_view & str)
void addToMapping(AccessType type, std::string_view str)
{
String str2{str};
boost::replace_all(str2, "_", " ");

4
src/Access/Common/AuthenticationData.cpp
View File

@ -71,7 +71,7 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty
}
AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::string_view & text [[maybe_unused]])
AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_view text [[maybe_unused]])
{
#if USE_SSL
Digest hash;
@ -86,7 +86,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::str
}
AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(const std::string_view & text)
AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(std::string_view text)
{
Poco::SHA1Engine engine;
engine.update(text.data(), text.size());

8
src/Access/Common/AuthenticationData.h
View File

@ -96,11 +96,11 @@ public:
struct Util
{
static Digest stringToDigest(const std::string_view & text) { return Digest(text.data(), text.data() + text.size()); }
static Digest encodeSHA256(const std::string_view & text);
static Digest encodeSHA1(const std::string_view & text);
static Digest stringToDigest(std::string_view text) { return Digest(text.data(), text.data() + text.size()); }
static Digest encodeSHA256(std::string_view text);
static Digest encodeSHA1(std::string_view text);
static Digest encodeSHA1(const Digest & text) { return encodeSHA1(std::string_view{reinterpret_cast<const char *>(text.data()), text.size()}); }
static Digest encodeDoubleSHA1(const std::string_view & text) { return encodeSHA1(encodeSHA1(text)); }
static Digest encodeDoubleSHA1(std::string_view text) { return encodeSHA1(encodeSHA1(text)); }
static Digest encodeDoubleSHA1(const Digest & text) { return encodeSHA1(encodeSHA1(text)); }
};

46
src/Access/ContextAccess.cpp
View File

@ -46,7 +46,7 @@ namespace
AccessRights addImplicitAccessRights(const AccessRights & access)
{
auto modifier = [&](const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, const std::string_view & database, const std::string_view & table, const std::string_view & column) -> AccessFlags
auto modifier = [&](const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, std::string_view database, std::string_view table, std::string_view column) -> AccessFlags
{
size_t level = !database.empty() + !table.empty() + !column.empty();
AccessFlags res = flags;
@ -141,7 +141,7 @@ namespace
std::string_view getDatabase() { return {}; }
template <typename... OtherArgs>
std::string_view getDatabase(const std::string_view & arg1, const OtherArgs &...) { return arg1; }
std::string_view getDatabase(std::string_view arg1, const OtherArgs &...) { return arg1; }
}
@ -519,7 +519,7 @@ bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const
}
template <bool throw_if_denied, bool grant_option, typename... Args>
bool ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const
bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const
{
return checkAccessImplHelper<throw_if_denied, grant_option>(flags, database.empty() ? params.current_database : database, args...);
}
@ -564,38 +564,38 @@ bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
}
bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl<false, false>(flags); }
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, false>(flags, database); }
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, false>(flags, database, table); }
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(flags, database); }
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(flags, database, table); }
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl<false, false>(element); }
bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(elements); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return checkAccessImpl<false, true>(flags); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, true>(flags, database); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, true>(flags, database, table); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(flags, database); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(flags, database, table); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return checkAccessImpl<false, true>(element); }
bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(elements); }
void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<true, false>(flags); }
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, false>(flags, database); }
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, false>(flags, database, table); }
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, false>(flags, database, table, column); }
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(flags, database); }
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(flags, database, table); }
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(flags, database, table, column); }
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<true, false>(element); }
void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<true, false>(elements); }
void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<true, true>(flags); }
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, true>(flags, database); }
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, true>(flags, database, table); }
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, true>(flags, database, table, column); }
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(flags, database); }
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(flags, database, table); }
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(flags, database, table, column); }
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<true, true>(element); }
void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true, true>(elements); }

42
src/Access/ContextAccess.h
View File

@ -101,40 +101,40 @@ public:
/// Checks if a specified access is granted, and throws an exception if not.
/// Empty database means the current database.
void checkAccess(const AccessFlags & flags) const;
void checkAccess(const AccessFlags & flags, const std::string_view & database) const;
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
void checkAccess(const AccessFlags & flags, std::string_view database) const;
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const;
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
void checkAccess(const AccessRightsElement & element) const;
void checkAccess(const AccessRightsElements & elements) const;
void checkGrantOption(const AccessFlags & flags) const;
void checkGrantOption(const AccessFlags & flags, const std::string_view & database) const;
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
void checkGrantOption(const AccessFlags & flags, std::string_view database) const;
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
void checkGrantOption(const AccessRightsElement & element) const;
void checkGrantOption(const AccessRightsElements & elements) const;
/// Checks if a specified access is granted, and returns false if not.
/// Empty database means the current database.
bool isGranted(const AccessFlags & flags) const;
bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
bool isGranted(const AccessFlags & flags, std::string_view database) const;
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
bool isGranted(const AccessRightsElement & element) const;
bool isGranted(const AccessRightsElements & elements) const;
bool hasGrantOption(const AccessFlags & flags) const;
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
bool hasGrantOption(const AccessRightsElement & element) const;
bool hasGrantOption(const AccessRightsElements & elements) const;
@ -180,7 +180,7 @@ private:
bool checkAccessImpl(const AccessFlags & flags) const;
template <bool throw_if_denied, bool grant_option, typename... Args>
bool checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const;
bool checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const;
template <bool throw_if_denied, bool grant_option>
bool checkAccessImpl(const AccessRightsElement & element) const;

20
src/Access/SettingsConstraints.cpp
View File

@ -36,12 +36,12 @@ void SettingsConstraints::clear()
}
void SettingsConstraints::setMinValue(const std::string_view & setting_name, const Field & min_value)
void SettingsConstraints::setMinValue(std::string_view setting_name, const Field & min_value)
{
getConstraintRef(setting_name).min_value = Settings::castValueUtil(setting_name, min_value);
}
Field SettingsConstraints::getMinValue(const std::string_view & setting_name) const
Field SettingsConstraints::getMinValue(std::string_view setting_name) const
{
const auto * ptr = tryGetConstraint(setting_name);
if (ptr)
@ -51,12 +51,12 @@ Field SettingsConstraints::getMinValue(const std::string_view & setting_name) co
}
void SettingsConstraints::setMaxValue(const std::string_view & setting_name, const Field & max_value)
void SettingsConstraints::setMaxValue(std::string_view setting_name, const Field & max_value)
{
getConstraintRef(setting_name).max_value = Settings::castValueUtil(setting_name, max_value);
}
Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) const
Field SettingsConstraints::getMaxValue(std::string_view setting_name) const
{
const auto * ptr = tryGetConstraint(setting_name);
if (ptr)
@ -66,12 +66,12 @@ Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) co
}
void SettingsConstraints::setReadOnly(const std::string_view & setting_name, bool read_only)
void SettingsConstraints::setReadOnly(std::string_view setting_name, bool read_only)
{
getConstraintRef(setting_name).read_only = read_only;
}
bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) const
bool SettingsConstraints::isReadOnly(std::string_view setting_name) const
{
const auto * ptr = tryGetConstraint(setting_name);
if (ptr)
@ -81,7 +81,7 @@ bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) cons
}
void SettingsConstraints::set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only)
void SettingsConstraints::set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only)
{
auto & ref = getConstraintRef(setting_name);
ref.min_value = Settings::castValueUtil(setting_name, min_value);
@ -89,7 +89,7 @@ void SettingsConstraints::set(const std::string_view & setting_name, const Field
ref.read_only = read_only;
}
void SettingsConstraints::get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const
void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const
{
const auto * ptr = tryGetConstraint(setting_name);
if (ptr)
@ -318,7 +318,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh
}
SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const std::string_view & setting_name)
SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(std::string_view setting_name)
{
auto it = constraints.find(setting_name);
if (it == constraints.end())
@ -331,7 +331,7 @@ SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const st
return it->second;
}
const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(const std::string_view & setting_name) const
const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(std::string_view setting_name) const
{
auto it = constraints.find(setting_name);
if (it == constraints.end())

20
src/Access/SettingsConstraints.h
View File

@ -61,17 +61,17 @@ public:
void clear();
bool empty() const { return constraints.empty(); }
void setMinValue(const std::string_view & setting_name, const Field & min_value);
Field getMinValue(const std::string_view & setting_name) const;
void setMinValue(std::string_view setting_name, const Field & min_value);
Field getMinValue(std::string_view setting_name) const;
void setMaxValue(const std::string_view & setting_name, const Field & max_value);
Field getMaxValue(const std::string_view & setting_name) const;
void setMaxValue(std::string_view setting_name, const Field & max_value);
Field getMaxValue(std::string_view setting_name) const;
void setReadOnly(const std::string_view & setting_name, bool read_only);
bool isReadOnly(const std::string_view & setting_name) const;
void setReadOnly(std::string_view setting_name, bool read_only);
bool isReadOnly(std::string_view setting_name) const;
void set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only);
void get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const;
void set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only);
void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const;
void merge(const SettingsConstraints & other);
@ -105,8 +105,8 @@ private:
};
bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const;
Constraint & getConstraintRef(const std::string_view & setting_name);
const Constraint * tryGetConstraint(const std::string_view & setting_name) const;
Constraint & getConstraintRef(std::string_view setting_name);
const Constraint * tryGetConstraint(std::string_view setting_name) const;
std::unordered_map<std::string_view, Constraint> constraints;
const AccessControl * access_control = nullptr;

6
src/Client/ClientBase.cpp
View File

@ -601,6 +601,7 @@ void ClientBase::initLogsOutputStream()
{
WriteBuffer * wb = out_logs_buf.get();
bool color_logs = false;
if (!out_logs_buf)
{
if (server_logs_file.empty())
@ -608,11 +609,13 @@ void ClientBase::initLogsOutputStream()
/// Use stderr by default
out_logs_buf = std::make_unique<WriteBufferFromFileDescriptor>(STDERR_FILENO);
wb = out_logs_buf.get();
color_logs = stderr_is_a_tty;
}
else if (server_logs_file == "-")
{
/// Use stdout if --server_logs_file=- specified
wb = &std_out;
color_logs = stdout_is_a_tty;
}
else
{
@ -622,7 +625,7 @@ void ClientBase::initLogsOutputStream()
}
}
logs_out_stream = std::make_unique<InternalTextLogs>(*wb, stdout_is_a_tty);
logs_out_stream = std::make_unique<InternalTextLogs>(*wb, color_logs);
}
}
@ -2155,6 +2158,7 @@ void ClientBase::init(int argc, char ** argv)
stdin_is_a_tty = isatty(STDIN_FILENO);
stdout_is_a_tty = isatty(STDOUT_FILENO);
stderr_is_a_tty = isatty(STDERR_FILENO);
terminal_width = getTerminalWidth();
Arguments common_arguments{""}; /// 0th argument is ignored.

1
src/Client/ClientBase.h
View File

@ -173,6 +173,7 @@ protected:
bool stdin_is_a_tty = false; /// stdin is a terminal.
bool stdout_is_a_tty = false; /// stdout is a terminal.
bool stderr_is_a_tty = false; /// stderr is a terminal.
uint64_t terminal_width = 0;
ServerConnectionPtr connection;

4
src/Common/JSONParsers/DummyJSONParser.h
View File

@ -84,7 +84,7 @@ struct DummyJSONParser
static Iterator begin() { return {}; }
static Iterator end() { return {}; }
static size_t size() { return 0; }
bool find(const std::string_view &, Element &) const { return false; } /// NOLINT
bool find(std::string_view, Element &) const { return false; } /// NOLINT
#if 0
/// Optional: Provides access to an object's element by index.
@ -93,7 +93,7 @@ struct DummyJSONParser
};
/// Parses a JSON document, returns the reference to its root element if succeeded.
bool parse(const std::string_view &, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
bool parse(std::string_view, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
#if 0
/// Optional: Allocates memory to parse JSON documents faster.

4
src/Common/JSONParsers/RapidJSONParser.h
View File

@ -98,7 +98,7 @@ struct RapidJSONParser
ALWAYS_INLINE Iterator end() const { return ptr->MemberEnd(); }
ALWAYS_INLINE size_t size() const { return ptr->MemberCount(); }
bool find(const std::string_view & key, Element & result) const
bool find(std::string_view key, Element & result) const
{
auto it = ptr->FindMember(rapidjson::StringRef(key.data(), key.length()));
if (it == ptr->MemberEnd())
@ -122,7 +122,7 @@ struct RapidJSONParser
};
/// Parses a JSON document, returns the reference to its root element if succeeded.
bool parse(const std::string_view & json, Element & result)
bool parse(std::string_view json, Element & result)
{
rapidjson::MemoryStream ms(json.data(), json.size());
rapidjson::EncodedInputStream<rapidjson::UTF8<>, rapidjson::MemoryStream> is(ms);

4
src/Common/JSONParsers/SimdJSONParser.h
View File

@ -105,7 +105,7 @@ struct SimdJSONParser
ALWAYS_INLINE Iterator end() const { return object.end(); }
ALWAYS_INLINE size_t size() const { return object.size(); }
bool find(const std::string_view & key, Element & result) const
bool find(std::string_view key, Element & result) const
{
auto x = object.at_key(key);
if (x.error())
@ -131,7 +131,7 @@ struct SimdJSONParser
};
/// Parses a JSON document, returns the reference to its root element if succeeded.
bool parse(const std::string_view & json, Element & result)
bool parse(std::string_view json, Element & result)
{
auto document = parser.parse(json.data(), json.size());
if (document.error())

4
src/Common/OpenSSLHelpers.cpp
View File

@ -10,7 +10,7 @@ namespace DB
{
#pragma GCC diagnostic warning "-Wold-style-cast"
std::string encodeSHA256(const std::string_view & text)
std::string encodeSHA256(std::string_view text)
{
return encodeSHA256(text.data(), text.size());
}
@ -21,7 +21,7 @@ std::string encodeSHA256(const void * text, size_t size)
encodeSHA256(text, size, reinterpret_cast<unsigned char *>(out.data()));
return out;
}
void encodeSHA256(const std::string_view & text, unsigned char * out)
void encodeSHA256(std::string_view text, unsigned char * out)
{
encodeSHA256(text.data(), text.size(), out);
}

4
src/Common/OpenSSLHelpers.h
View File

@ -10,10 +10,10 @@ namespace DB
{
/// Encodes `text` and returns it.
std::string encodeSHA256(const std::string_view & text);
std::string encodeSHA256(std::string_view text);
std::string encodeSHA256(const void * text, size_t size);
/// `out` must be at least 32 bytes long.
void encodeSHA256(const std::string_view & text, unsigned char * out);
void encodeSHA256(std::string_view text, unsigned char * out);
void encodeSHA256(const void * text, size_t size, unsigned char * out);
/// Returns concatenation of error strings for all errors that OpenSSL has recorded, emptying the error queue.

10
src/Common/SettingsChanges.cpp
View File

@ -4,7 +4,7 @@ namespace DB
{
namespace
{
SettingChange * find(SettingsChanges & changes, const std::string_view & name)
SettingChange * find(SettingsChanges & changes, std::string_view name)
{
auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
if (it == changes.end())
@ -12,7 +12,7 @@ namespace
return &*it;
}
const SettingChange * find(const SettingsChanges & changes, const std::string_view & name)
const SettingChange * find(const SettingsChanges & changes, std::string_view name)
{
auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
if (it == changes.end())
@ -21,7 +21,7 @@ namespace
}
}
bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) const
bool SettingsChanges::tryGet(std::string_view name, Field & out_value) const
{
const auto * change = find(*this, name);
if (!change)
@ -30,7 +30,7 @@ bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) c
return true;
}
const Field * SettingsChanges::tryGet(const std::string_view & name) const
const Field * SettingsChanges::tryGet(std::string_view name) const
{
const auto * change = find(*this, name);
if (!change)
@ -38,7 +38,7 @@ const Field * SettingsChanges::tryGet(const std::string_view & name) const
return &change->value;
}
Field * SettingsChanges::tryGet(const std::string_view & name)
Field * SettingsChanges::tryGet(std::string_view name)
{
auto * change = find(*this, name);
if (!change)

10
src/Common/SettingsChanges.h
View File

@ -14,8 +14,8 @@ struct SettingChange
Field value;
SettingChange() = default;
SettingChange(const std::string_view & name_, const Field & value_) : name(name_), value(value_) {}
SettingChange(const std::string_view & name_, Field && value_) : name(name_), value(std::move(value_)) {}
SettingChange(std::string_view name_, const Field & value_) : name(name_), value(value_) {}
SettingChange(std::string_view name_, Field && value_) : name(name_), value(std::move(value_)) {}
friend bool operator ==(const SettingChange & lhs, const SettingChange & rhs) { return (lhs.name == rhs.name) && (lhs.value == rhs.value); }
friend bool operator !=(const SettingChange & lhs, const SettingChange & rhs) { return !(lhs == rhs); }
@ -27,9 +27,9 @@ class SettingsChanges : public std::vector<SettingChange>
public:
using std::vector<SettingChange>::vector;
bool tryGet(const std::string_view & name, Field & out_value) const;
const Field * tryGet(const std::string_view & name) const;
Field * tryGet(const std::string_view & name);
bool tryGet(std::string_view name, Field & out_value) const;
const Field * tryGet(std::string_view name) const;
Field * tryGet(std::string_view name);
};
}

2
src/Common/StringUtils/StringUtils.h
View File

@ -147,7 +147,7 @@ inline bool isPunctuationASCII(char c)
}
inline bool isValidIdentifier(const std::string_view & str)
inline bool isValidIdentifier(std::string_view str)
{
return !str.empty()
&& isValidIdentifierBegin(str[0])

1
src/Common/TaskStatsInfoGetter.cpp
View File

@ -21,6 +21,7 @@
#if defined(__clang__)
#pragma clang diagnostic ignored "-Wgnu-anonymous-struct"
#pragma clang diagnostic ignored "-Wnested-anon-types"
#endif
/// Basic idea is motivated by "iotop" tool.

5
src/Common/Throttler.h
View File

@ -1,5 +1,7 @@
#pragma once
#include <Common/Throttler_fwd.h>
#include <mutex>
#include <memory>
#include <base/sleep.h>
@ -57,7 +59,4 @@ private:
std::shared_ptr<Throttler> parent;
};
using ThrottlerPtr = std::shared_ptr<Throttler>;
}

11
src/Common/Throttler_fwd.h Normal file
View File

@ -0,0 +1,11 @@
#pragma once
#include <memory>
namespace DB
{
class Throttler;
using ThrottlerPtr = std::shared_ptr<Throttler>;
}

46
src/Common/noexcept_scope.h
View File

@ -1,36 +1,28 @@
#pragma once
#include <base/scope_guard.h>
#include <Common/Exception.h>
#include <Common/LockMemoryExceptionInThread.h>
#define NOEXCEPT_SCOPE_IMPL_CONCAT(n, expected) \
LockMemoryExceptionInThread lock_memory_tracker##n(VariableContext::Global); \
SCOPE_EXIT( \
{ \
const auto uncaught = std::uncaught_exceptions(); \
assert((expected) == uncaught || (expected) + 1 == uncaught); \
if ((expected) < uncaught) \
{ \
tryLogCurrentException("NOEXCEPT_SCOPE"); \
abort(); \
} \
} \
)
#define NOEXCEPT_SCOPE_IMPL(n, expected) NOEXCEPT_SCOPE_IMPL_CONCAT(n, expected)
#define NOEXCEPT_SCOPE_CONCAT(n) \
const auto num_curr_exceptions##n = std::uncaught_exceptions(); \
NOEXCEPT_SCOPE_IMPL(n, num_curr_exceptions##n)
#define NOEXCEPT_SCOPE_FWD(n) NOEXCEPT_SCOPE_CONCAT(n)
/// It can be used in critical places to exit on unexpected exceptions.
/// SIGABRT is usually better that broken in-memory state with unpredictable consequences.
/// It also temporarily disables exception from memory tracker in current thread.
/// Strict version does not take into account nested exception (i.e. it aborts even when we're in catch block).
#define NOEXCEPT_SCOPE_STRICT NOEXCEPT_SCOPE_IMPL(__LINE__, 0)
#define NOEXCEPT_SCOPE NOEXCEPT_SCOPE_FWD(__LINE__)
#define NOEXCEPT_SCOPE_IMPL(...) do { \
LockMemoryExceptionInThread \
noexcept_lock_memory_tracker(VariableContext::Global); \
try \
{ \
__VA_ARGS__; \
} \
catch (...) \
{ \
DB::tryLogCurrentException(__PRETTY_FUNCTION__); \
std::terminate(); \
} \
} while (0) /* to allow leading semi-colon */
#define NOEXCEPT_SCOPE_STRICT(...) \
if (std::uncaught_exceptions()) std::terminate(); \
NOEXCEPT_SCOPE_IMPL(__VA_ARGS__)
#define NOEXCEPT_SCOPE(...) NOEXCEPT_SCOPE_IMPL(__VA_ARGS__)

4
src/Compression/CompressionCodecEncrypted.cpp
View File

@ -131,7 +131,7 @@ std::string lastErrorString()
/// This function get key and nonce and encrypt text with their help.
/// If something went wrong (can't init context or can't encrypt data) it throws exception.
/// It returns length of encrypted text.
size_t encrypt(const std::string_view & plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce)
size_t encrypt(std::string_view plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce)
{
/// Init context for encryption, using key.
EVP_AEAD_CTX encrypt_ctx;
@ -160,7 +160,7 @@ size_t encrypt(const std::string_view & plaintext, char * ciphertext_and_tag, En
/// This function get key and nonce and encrypt text with their help.
/// If something went wrong (can't init context or can't encrypt data) it throws exception.
/// It returns length of encrypted text.
size_t decrypt(const std::string_view & ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce)
size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce)
{
/// Init context for decryption with given key.
EVP_AEAD_CTX decrypt_ctx;

6
src/Core/BaseSettings.cpp
View File

@ -11,7 +11,7 @@ namespace ErrorCodes
extern const int UNKNOWN_SETTING;
}
void BaseSettingsHelpers::writeString(const std::string_view & str, WriteBuffer & out)
void BaseSettingsHelpers::writeString(std::string_view str, WriteBuffer & out)
{
writeStringBinary(str, out);
}
@ -39,13 +39,13 @@ BaseSettingsHelpers::Flags BaseSettingsHelpers::readFlags(ReadBuffer & in)
}
void BaseSettingsHelpers::throwSettingNotFound(const std::string_view & name)
void BaseSettingsHelpers::throwSettingNotFound(std::string_view name)
{
throw Exception("Unknown setting " + String{name}, ErrorCodes::UNKNOWN_SETTING);
}
void BaseSettingsHelpers::warningSettingNotFound(const std::string_view & name)
void BaseSettingsHelpers::warningSettingNotFound(std::string_view name)
{
static auto * log = &Poco::Logger::get("Settings");
LOG_WARNING(log, "Unknown setting {}, skipping", name);

88
src/Core/BaseSettings.h
View File

@ -45,16 +45,16 @@ class BaseSettings : public TTraits::Data
public:
using Traits = TTraits;
void set(const std::string_view & name, const Field & value);
Field get(const std::string_view & name) const;
void set(std::string_view name, const Field & value);
Field get(std::string_view name) const;
void setString(const std::string_view & name, const String & value);
String getString(const std::string_view & name) const;
void setString(std::string_view name, const String & value);
String getString(std::string_view name) const;
bool tryGet(const std::string_view & name, Field & value) const;
bool tryGetString(const std::string_view & name, String & value) const;
bool tryGet(std::string_view name, Field & value) const;
bool tryGetString(std::string_view name, String & value) const;
bool isChanged(const std::string_view & name) const;
bool isChanged(std::string_view name) const;
SettingsChanges changes() const;
void applyChange(const SettingChange & change);
void applyChanges(const SettingsChanges & changes);
@ -63,22 +63,22 @@ public:
/// Resets all the settings to their default values.
void resetToDefault();
bool has(const std::string_view & name) const { return hasBuiltin(name) || hasCustom(name); }
static bool hasBuiltin(const std::string_view & name);
bool hasCustom(const std::string_view & name) const;
bool has(std::string_view name) const { return hasBuiltin(name) || hasCustom(name); }
static bool hasBuiltin(std::string_view name);
bool hasCustom(std::string_view name) const;
const char * getTypeName(const std::string_view & name) const;
const char * getDescription(const std::string_view & name) const;
const char * getTypeName(std::string_view name) const;
const char * getDescription(std::string_view name) const;
/// Checks if it's possible to assign a field to a specified value and throws an exception if not.
/// This function doesn't change the fields, it performs check only.
static void checkCanSet(const std::string_view & name, const Field & value);
static void checkCanSetString(const std::string_view & name, const String & str);
static void checkCanSet(std::string_view name, const Field & value);
static void checkCanSetString(std::string_view name, const String & str);
/// Conversions without changing the fields.
static Field castValueUtil(const std::string_view & name, const Field & value);
static String valueToStringUtil(const std::string_view & name, const Field & value);
static Field stringToValueUtil(const std::string_view & name, const String & str);
static Field castValueUtil(std::string_view name, const Field & value);
static String valueToStringUtil(std::string_view name, const Field & value);
static Field stringToValueUtil(std::string_view name, const String & str);
void write(WriteBuffer & out, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT) const;
void read(ReadBuffer & in, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT);
@ -164,19 +164,19 @@ public:
Iterator end() const { return allChanged().end(); }
private:
SettingFieldCustom & getCustomSetting(const std::string_view & name);
const SettingFieldCustom & getCustomSetting(const std::string_view & name) const;
const SettingFieldCustom * tryGetCustomSetting(const std::string_view & name) const;
SettingFieldCustom & getCustomSetting(std::string_view name);
const SettingFieldCustom & getCustomSetting(std::string_view name) const;
const SettingFieldCustom * tryGetCustomSetting(std::string_view name) const;
std::conditional_t<Traits::allow_custom_settings, CustomSettingMap, boost::blank> custom_settings_map;
};
struct BaseSettingsHelpers
{
[[noreturn]] static void throwSettingNotFound(const std::string_view & name);
static void warningSettingNotFound(const std::string_view & name);
[[noreturn]] static void throwSettingNotFound(std::string_view name);
static void warningSettingNotFound(std::string_view name);
static void writeString(const std::string_view & str, WriteBuffer & out);
static void writeString(std::string_view str, WriteBuffer & out);
static String readString(ReadBuffer & in);
enum Flags : UInt64
@ -190,7 +190,7 @@ struct BaseSettingsHelpers
};
template <typename TTraits>
void BaseSettings<TTraits>::set(const std::string_view & name, const Field & value)
void BaseSettings<TTraits>::set(std::string_view name, const Field & value)
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -200,7 +200,7 @@ void BaseSettings<TTraits>::set(const std::string_view & name, const Field & val
}
template <typename TTraits>
Field BaseSettings<TTraits>::get(const std::string_view & name) const
Field BaseSettings<TTraits>::get(std::string_view name) const
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -210,7 +210,7 @@ Field BaseSettings<TTraits>::get(const std::string_view & name) const
}
template <typename TTraits>
void BaseSettings<TTraits>::setString(const std::string_view & name, const String & value)
void BaseSettings<TTraits>::setString(std::string_view name, const String & value)
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -220,7 +220,7 @@ void BaseSettings<TTraits>::setString(const std::string_view & name, const Strin
}
template <typename TTraits>
String BaseSettings<TTraits>::getString(const std::string_view & name) const
String BaseSettings<TTraits>::getString(std::string_view name) const
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -230,7 +230,7 @@ String BaseSettings<TTraits>::getString(const std::string_view & name) const
}
template <typename TTraits>
bool BaseSettings<TTraits>::tryGet(const std::string_view & name, Field & value) const
bool BaseSettings<TTraits>::tryGet(std::string_view name, Field & value) const
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -247,7 +247,7 @@ bool BaseSettings<TTraits>::tryGet(const std::string_view & name, Field & value)
}
template <typename TTraits>
bool BaseSettings<TTraits>::tryGetString(const std::string_view & name, String & value) const
bool BaseSettings<TTraits>::tryGetString(std::string_view name, String & value) const
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -264,7 +264,7 @@ bool BaseSettings<TTraits>::tryGetString(const std::string_view & name, String &
}
template <typename TTraits>
bool BaseSettings<TTraits>::isChanged(const std::string_view & name) const
bool BaseSettings<TTraits>::isChanged(std::string_view name) const
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -316,20 +316,20 @@ void BaseSettings<TTraits>::resetToDefault()
}
template <typename TTraits>
bool BaseSettings<TTraits>::hasBuiltin(const std::string_view & name)
bool BaseSettings<TTraits>::hasBuiltin(std::string_view name)
{
const auto & accessor = Traits::Accessor::instance();
return (accessor.find(name) != static_cast<size_t>(-1));
}
template <typename TTraits>
bool BaseSettings<TTraits>::hasCustom(const std::string_view & name) const
bool BaseSettings<TTraits>::hasCustom(std::string_view name) const
{
return tryGetCustomSetting(name);
}
template <typename TTraits>
const char * BaseSettings<TTraits>::getTypeName(const std::string_view & name) const
const char * BaseSettings<TTraits>::getTypeName(std::string_view name) const
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -341,7 +341,7 @@ const char * BaseSettings<TTraits>::getTypeName(const std::string_view & name) c
}
template <typename TTraits>
const char * BaseSettings<TTraits>::getDescription(const std::string_view & name) const
const char * BaseSettings<TTraits>::getDescription(std::string_view name) const
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -353,19 +353,19 @@ const char * BaseSettings<TTraits>::getDescription(const std::string_view & name
}
template <typename TTraits>
void BaseSettings<TTraits>::checkCanSet(const std::string_view & name, const Field & value)
void BaseSettings<TTraits>::checkCanSet(std::string_view name, const Field & value)
{
castValueUtil(name, value);
}
template <typename TTraits>
void BaseSettings<TTraits>::checkCanSetString(const std::string_view & name, const String & str)
void BaseSettings<TTraits>::checkCanSetString(std::string_view name, const String & str)
{
stringToValueUtil(name, str);
}
template <typename TTraits>
Field BaseSettings<TTraits>::castValueUtil(const std::string_view & name, const Field & value)
Field BaseSettings<TTraits>::castValueUtil(std::string_view name, const Field & value)
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -377,7 +377,7 @@ Field BaseSettings<TTraits>::castValueUtil(const std::string_view & name, const
}
template <typename TTraits>
String BaseSettings<TTraits>::valueToStringUtil(const std::string_view & name, const Field & value)
String BaseSettings<TTraits>::valueToStringUtil(std::string_view name, const Field & value)
{
const auto & accessor = Traits::Accessor::instance();
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
@ -389,7 +389,7 @@ String BaseSettings<TTraits>::valueToStringUtil(const std::string_view & name, c
}
template <typename TTraits>
Field BaseSettings<TTraits>::stringToValueUtil(const std::string_view & name, const String & str)
Field BaseSettings<TTraits>::stringToValueUtil(std::string_view name, const String & str)
{
try
{
@ -521,7 +521,7 @@ bool operator!=(const BaseSettings<TTraits> & left, const BaseSettings<TTraits>
}
template <typename TTraits>
SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_view & name)
SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(std::string_view name)
{
if constexpr (Traits::allow_custom_settings)
{
@ -537,7 +537,7 @@ SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_v
}
template <typename TTraits>
const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_view & name) const
const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(std::string_view name) const
{
if constexpr (Traits::allow_custom_settings)
{
@ -549,7 +549,7 @@ const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::st
}
template <typename TTraits>
const SettingFieldCustom * BaseSettings<TTraits>::tryGetCustomSetting(const std::string_view & name) const
const SettingFieldCustom * BaseSettings<TTraits>::tryGetCustomSetting(std::string_view name) const
{
if constexpr (Traits::allow_custom_settings)
{
@ -780,7 +780,7 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
public: \
static const Accessor & instance(); \
size_t size() const { return field_infos.size(); } \
size_t find(const std::string_view & name) const; \
size_t find(std::string_view name) const; \
const String & getName(size_t index) const { return field_infos[index].name; } \
const char * getTypeName(size_t index) const { return field_infos[index].type; } \
const char * getDescription(size_t index) const { return field_infos[index].description; } \
@ -851,7 +851,7 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
\
SETTINGS_TRAITS_NAME::Accessor::Accessor() {} \
\
size_t SETTINGS_TRAITS_NAME::Accessor::find(const std::string_view & name) const \
size_t SETTINGS_TRAITS_NAME::Accessor::find(std::string_view name) const \
{ \
auto it = name_to_index_map.find(name); \
if (it != name_to_index_map.end()) \

2
src/Core/Field.cpp
View File

@ -286,7 +286,7 @@ String Field::dump() const
return applyVisitor(FieldVisitorDump(), *this);
}
Field Field::restoreFromDump(const std::string_view & dump_)
Field Field::restoreFromDump(std::string_view dump_)
{
auto show_error = [&dump_]
{

8
src/Core/Field.h
View File

@ -346,7 +346,7 @@ public:
}
/// Create a string inplace.
Field(const std::string_view & str) { create(str.data(), str.size()); } /// NOLINT
Field(std::string_view str) { create(str.data(), str.size()); } /// NOLINT
Field(const String & str) { create(std::string_view{str}); } /// NOLINT
Field(String && str) { create(std::move(str)); } /// NOLINT
Field(const char * str) { create(std::string_view{str}); } /// NOLINT
@ -403,7 +403,7 @@ public:
return *this;
}
Field & operator= (const std::string_view & str);
Field & operator= (std::string_view str);
Field & operator= (const String & str) { return *this = std::string_view{str}; }
Field & operator= (String && str);
Field & operator= (const char * str) { return *this = std::string_view{str}; }
@ -631,7 +631,7 @@ public:
}
String dump() const;
static Field restoreFromDump(const std::string_view & dump_);
static Field restoreFromDump(std::string_view dump_);
private:
std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(Types::Which),
@ -929,7 +929,7 @@ Field::operator=(T && rhs)
return *this;
}
inline Field & Field::operator=(const std::string_view & str)
inline Field & Field::operator=(std::string_view str)
{
if (which != Types::String)
{

2
src/Core/Settings.h
View File

@ -96,6 +96,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \
M(UInt64, max_replicated_fetches_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated fetches. Zero means unlimited. Only has meaning at server startup.", 0) \
M(UInt64, max_replicated_sends_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited. Only has meaning at server startup.", 0) \
M(UInt64, max_remote_read_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for read. Zero means unlimited. Only has meaning at server startup.", 0) \
M(UInt64, max_remote_write_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for write. Zero means unlimited. Only has meaning at server startup.", 0) \
M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \
M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \
\

2
src/Core/SettingsFields.cpp
View File

@ -350,7 +350,7 @@ void SettingFieldURI::readBinary(ReadBuffer & in)
}
void SettingFieldEnumHelpers::writeBinary(const std::string_view & str, WriteBuffer & out)
void SettingFieldEnumHelpers::writeBinary(std::string_view str, WriteBuffer & out)
{
writeStringBinary(str, out);
}

12
src/Core/SettingsFields.h
View File

@ -146,13 +146,13 @@ struct SettingFieldString
String value;
bool changed = false;
explicit SettingFieldString(const std::string_view & str = {}) : value(str) {}
explicit SettingFieldString(std::string_view str = {}) : value(str) {}
explicit SettingFieldString(const String & str) : SettingFieldString(std::string_view{str}) {}
explicit SettingFieldString(String && str) : value(std::move(str)) {}
explicit SettingFieldString(const char * str) : SettingFieldString(std::string_view{str}) {}
explicit SettingFieldString(const Field & f) : SettingFieldString(f.safeGet<const String &>()) {}
SettingFieldString & operator =(const std::string_view & str) { value = str; changed = true; return *this; }
SettingFieldString & operator =(std::string_view str) { value = str; changed = true; return *this; }
SettingFieldString & operator =(const String & str) { *this = std::string_view{str}; return *this; }
SettingFieldString & operator =(String && str) { value = std::move(str); changed = true; return *this; }
SettingFieldString & operator =(const char * str) { *this = std::string_view{str}; return *this; }
@ -256,7 +256,7 @@ struct SettingFieldEnum
struct SettingFieldEnumHelpers
{
static void writeBinary(const std::string_view & str, WriteBuffer & out);
static void writeBinary(std::string_view str, WriteBuffer & out);
static String readBinary(ReadBuffer & in);
};
@ -286,7 +286,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
{ \
using EnumType = ENUM_TYPE; \
static const String & toString(EnumType value); \
static EnumType fromString(const std::string_view & str); \
static EnumType fromString(std::string_view str); \
}; \
\
using SettingField##NEW_NAME = SettingFieldEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;
@ -310,7 +310,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
ERROR_CODE_FOR_UNEXPECTED_NAME); \
} \
\
typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(const std::string_view & str) \
typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(std::string_view str) \
{ \
static const std::unordered_map<std::string_view, EnumType> map = [] { \
std::unordered_map<std::string_view, EnumType> res; \
@ -430,7 +430,7 @@ void SettingFieldMultiEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
using EnumType = ENUM_TYPE; \
static size_t getEnumSize(); \
static const String & toString(EnumType value); \
static EnumType fromString(const std::string_view & str); \
static EnumType fromString(std::string_view str); \
}; \
\
using SettingField##NEW_NAME = SettingFieldMultiEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;

2
src/DataTypes/NestedUtils.cpp
View File

@ -54,7 +54,7 @@ std::pair<std::string, std::string> splitName(const std::string & name, bool rev
return {name.substr(0, idx), name.substr(idx + 1)};
}
std::pair<std::string_view, std::string_view> splitName(const std::string_view & name, bool reverse)
std::pair<std::string_view, std::string_view> splitName(std::string_view name, bool reverse)
{
auto idx = (reverse ? name.find_last_of('.') : name.find_first_of('.'));
if (idx == std::string::npos || idx == 0 || idx + 1 == name.size())

2
src/DataTypes/NestedUtils.h
View File

@ -13,7 +13,7 @@ namespace Nested
/// Splits name of compound identifier by first/last dot (depending on 'reverse' parameter).
std::pair<std::string, std::string> splitName(const std::string & name, bool reverse = false);
std::pair<std::string_view, std::string_view> splitName(const std::string_view & name, bool reverse = false);
std::pair<std::string_view, std::string_view> splitName(std::string_view name, bool reverse = false);
/// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot.
std::string extractTableName(const std::string & nested_name);

1
src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
View File

@ -90,7 +90,6 @@ SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementation
settings,
max_single_read_retries,
max_single_download_retries,
settings.remote_fs_buffer_size,
/* use_external_buffer */true,
read_until_position);
}

18
src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
View File

@ -67,8 +67,8 @@ std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObject( /// NOLI
auto settings_ptr = settings.get();
return std::make_unique<ReadBufferFromAzureBlobStorage>(
client.get(), object.absolute_path, read_settings, settings_ptr->max_single_read_retries,
settings_ptr->max_single_download_retries, read_settings.remote_fs_buffer_size);
client.get(), object.absolute_path, patchSettings(read_settings), settings_ptr->max_single_read_retries,
settings_ptr->max_single_download_retries);
}
std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOLINT
@ -77,18 +77,19 @@ std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOL
std::optional<size_t>,
std::optional<size_t>) const
{
ReadSettings disk_read_settings = patchSettings(read_settings);
auto settings_ptr = settings.get();
auto reader_impl = std::make_unique<ReadBufferFromAzureBlobStorageGather>(
client.get(),
objects,
settings_ptr->max_single_read_retries,
settings_ptr->max_single_download_retries,
read_settings);
disk_read_settings);
if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
if (disk_read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
{
auto reader = getThreadPoolReader();
return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, read_settings, std::move(reader_impl));
return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, disk_read_settings, std::move(reader_impl));
}
else
{
@ -104,7 +105,7 @@ std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NO
std::optional<ObjectAttributes>,
FinalizeCallback && finalize_callback,
size_t buf_size,
const WriteSettings &)
const WriteSettings & write_settings)
{
if (mode != WriteMode::Rewrite)
throw Exception("Azure storage doesn't support append", ErrorCodes::UNSUPPORTED_METHOD);
@ -113,7 +114,8 @@ std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NO
client.get(),
object.absolute_path,
settings.get()->max_single_part_upload_size,
buf_size);
buf_size,
patchSettings(write_settings));
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(finalize_callback), object.absolute_path);
}
@ -207,7 +209,7 @@ void AzureObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguratio
{
auto new_settings = getAzureBlobStorageSettings(config, config_prefix, context);
settings.set(std::move(new_settings));
applyRemoteThrottlingSettings(context);
/// We don't update client
}

11
src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
View File

@ -52,7 +52,7 @@ std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObject( /// NOLIN
std::optional<size_t>,
std::optional<size_t>) const
{
return std::make_unique<ReadBufferFromHDFS>(object.absolute_path, object.absolute_path, config, read_settings);
return std::make_unique<ReadBufferFromHDFS>(object.absolute_path, object.absolute_path, config, patchSettings(read_settings));
}
std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObjects( /// NOLINT
@ -61,7 +61,7 @@ std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObjects( /// NOLI
std::optional<size_t>,
std::optional<size_t>) const
{
auto hdfs_impl = std::make_unique<ReadBufferFromHDFSGather>(config, objects, read_settings);
auto hdfs_impl = std::make_unique<ReadBufferFromHDFSGather>(config, objects, patchSettings(read_settings));
auto buf = std::make_unique<ReadIndirectBufferFromRemoteFS>(std::move(hdfs_impl));
return std::make_unique<SeekAvoidingReadBuffer>(std::move(buf), settings->min_bytes_for_seek);
}
@ -72,7 +72,7 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
std::optional<ObjectAttributes> attributes,
FinalizeCallback && finalize_callback,
size_t buf_size,
const WriteSettings &)
const WriteSettings & write_settings)
{
if (attributes.has_value())
throw Exception(
@ -81,7 +81,7 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
/// Single O_WRONLY in libhdfs adds O_TRUNC
auto hdfs_buffer = std::make_unique<WriteBufferFromHDFS>(
object.absolute_path, config, settings->replication, buf_size,
object.absolute_path, config, settings->replication, patchSettings(write_settings), buf_size,
mode == WriteMode::Rewrite ? O_WRONLY : O_WRONLY | O_APPEND);
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(hdfs_buffer), std::move(finalize_callback), object.absolute_path);
@ -155,8 +155,9 @@ void HDFSObjectStorage::copyObject( /// NOLINT
}
void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr)
void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr context)
{
applyRemoteThrottlingSettings(context);
}
std::unique_ptr<IObjectStorage> HDFSObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr)

23
src/Disks/ObjectStorages/IObjectStorage.cpp
View File

@ -47,4 +47,27 @@ std::string IObjectStorage::getCacheBasePath() const
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getCacheBasePath() is not implemented for {}", getName());
}
void IObjectStorage::applyRemoteThrottlingSettings(ContextPtr context)
{
std::unique_lock lock{throttlers_mutex};
remote_read_throttler = context->getRemoteReadThrottler();
remote_write_throttler = context->getRemoteWriteThrottler();
}
ReadSettings IObjectStorage::patchSettings(const ReadSettings & read_settings) const
{
std::unique_lock lock{throttlers_mutex};
ReadSettings settings{read_settings};
settings.remote_throttler = remote_read_throttler;
return settings;
}
WriteSettings IObjectStorage::patchSettings(const WriteSettings & write_settings) const
{
std::unique_lock lock{throttlers_mutex};
WriteSettings settings{write_settings};
settings.remote_throttler = remote_write_throttler;
return settings;
}
}

14
src/Disks/ObjectStorages/IObjectStorage.h
View File

@ -3,6 +3,7 @@
#include <filesystem>
#include <string>
#include <map>
#include <mutex>
#include <optional>
#include <Poco/Timestamp.h>
@ -166,6 +167,19 @@ public:
virtual void removeCacheIfExists(const std::string & /* path */) {}
virtual bool supportsCache() const { return false; }
protected:
/// Should be called from implementation of applyNewSettings()
void applyRemoteThrottlingSettings(ContextPtr context);
/// Should be used by implementation of read* and write* methods
ReadSettings patchSettings(const ReadSettings & read_settings) const;
WriteSettings patchSettings(const WriteSettings & write_settings) const;
private:
mutable std::mutex throttlers_mutex;
ThrottlerPtr remote_read_throttler;
ThrottlerPtr remote_write_throttler;
};
using ObjectStoragePtr = std::shared_ptr<IObjectStorage>;

29
src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
View File

@ -139,14 +139,7 @@ std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObjects( /// NOLINT
{
assert(!objects[0].getPathKeyForCache().empty());
ReadSettings disk_read_settings{read_settings};
if (cache)
{
if (IFileCache::isReadOnly())
disk_read_settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = true;
disk_read_settings.remote_fs_cache = cache;
}
ReadSettings disk_read_settings = patchSettings(read_settings);
auto settings_ptr = s3_settings.get();
@ -183,10 +176,9 @@ std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObject( /// NOLINT
object.absolute_path,
version_id,
settings_ptr->s3_settings.max_single_read_retries,
read_settings);
patchSettings(read_settings));
}
std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLINT
const StoredObject & object,
WriteMode mode, // S3 doesn't support append, only rewrite
@ -195,6 +187,8 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
size_t buf_size,
const WriteSettings & write_settings)
{
WriteSettings disk_write_settings = IObjectStorage::patchSettings(write_settings);
if (mode != WriteMode::Rewrite)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "S3 doesn't support append to files");
@ -211,6 +205,7 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
attributes,
buf_size,
threadPoolCallbackRunner(getThreadPoolWriter()),
disk_write_settings,
cache_on_write ? cache : nullptr);
@ -485,6 +480,19 @@ void S3ObjectStorage::copyObject( // NOLINT
}
}
ReadSettings S3ObjectStorage::patchSettings(const ReadSettings & read_settings) const
{
ReadSettings settings{read_settings};
if (cache)
{
if (IFileCache::isReadOnly())
settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = true;
settings.remote_fs_cache = cache;
}
return IObjectStorage::patchSettings(settings);
}
void S3ObjectStorage::setNewSettings(std::unique_ptr<S3ObjectStorageSettings> && s3_settings_)
{
s3_settings.set(std::move(s3_settings_));
@ -517,6 +525,7 @@ void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &
{
s3_settings.set(getSettings(config, config_prefix, context));
client.set(getClient(config, config_prefix, context));
applyRemoteThrottlingSettings(context);
}
std::unique_ptr<IObjectStorage> S3ObjectStorage::cloneObjectStorage(

2
src/Disks/ObjectStorages/S3/S3ObjectStorage.h
View File

@ -142,6 +142,8 @@ public:
String getCacheBasePath() const override;
private:
ReadSettings patchSettings(const ReadSettings & read_settings) const;
void setNewSettings(std::unique_ptr<S3ObjectStorageSettings> && s3_settings_);
void setNewClient(std::unique_ptr<Aws::S3::S3Client> && client_);

88
src/Formats/ProtobufSerializer.cpp
View File

@ -77,18 +77,18 @@ namespace
return convertChar(c1) == convertChar(c2);
}
static bool equals(const std::string_view & s1, const std::string_view & s2)
static bool equals(std::string_view s1, std::string_view s2)
{
return (s1.length() == s2.length())
&& std::equal(s1.begin(), s1.end(), s2.begin(), [](char c1, char c2) { return convertChar(c1) == convertChar(c2); });
}
static bool less(const std::string_view & s1, const std::string_view & s2)
static bool less(std::string_view s1, std::string_view s2)
{
return std::lexicographical_compare(s1.begin(), s1.end(), s2.begin(), s2.end(), [](char c1, char c2) { return convertChar(c1) < convertChar(c2); });
}
static bool startsWith(const std::string_view & s1, const std::string_view & s2)
static bool startsWith(std::string_view s1, std::string_view s2)
{
return (s1.length() >= s2.length()) && equals(s1.substr(0, s2.length()), s2);
}
@ -195,7 +195,7 @@ namespace
{
protected:
ProtobufSerializerSingleValue(
const std::string_view & column_name_,
std::string_view column_name_,
const FieldDescriptor & field_descriptor_,
const ProtobufReaderOrWriter & reader_or_writer_)
: column_name(column_name_)
@ -264,7 +264,7 @@ namespace
return reader->readFixed<FieldType>();
}
void writeStr(const std::string_view & str)
void writeStr(std::string_view str)
{
if (!str.empty() || !skip_zero_or_empty)
writer->writeString(field_tag, str);
@ -274,7 +274,7 @@ namespace
void readStrAndAppend(PaddedPODArray<UInt8> & str) { reader->readStringAndAppend(str); }
template <typename DestType>
DestType parseFromStr(const std::string_view & str) const
DestType parseFromStr(std::string_view str) const
{
try
{
@ -307,7 +307,7 @@ namespace
return result;
}
[[noreturn]] void incompatibleColumnType(const std::string_view & column_type) const
[[noreturn]] void incompatibleColumnType(std::string_view column_type) const
{
throw Exception(
ErrorCodes::DATA_TYPE_INCOMPATIBLE_WITH_PROTOBUF_FIELD,
@ -318,7 +318,7 @@ namespace
field_descriptor.type_name());
}
[[noreturn]] void cannotConvertValue(const std::string_view & src_value, const std::string_view & src_type_name, const std::string_view & dest_type_name) const
[[noreturn]] void cannotConvertValue(std::string_view src_value, std::string_view src_type_name, std::string_view dest_type_name) const
{
throw Exception(
"Could not convert value '" + String{src_value} + "' from type " + String{src_type_name} + " to type "
@ -351,7 +351,7 @@ namespace
public:
using ColumnType = ColumnVector<NumberType>;
ProtobufSerializerNumber(const std::string_view & column_name_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_)
ProtobufSerializerNumber(std::string_view column_name_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_)
: ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_)
{
setFunctions();
@ -590,7 +590,7 @@ namespace
using ColumnType = std::conditional_t<is_fixed_string, ColumnFixedString, ColumnString>;
ProtobufSerializerString(
const std::string_view & column_name_,
std::string_view column_name_,
const std::shared_ptr<const DataTypeFixedString> & fixed_string_data_type_,
const google::protobuf::FieldDescriptor & field_descriptor_,
const ProtobufReaderOrWriter & reader_or_writer_)
@ -604,7 +604,7 @@ namespace
}
ProtobufSerializerString(
const std::string_view & column_name_,
std::string_view column_name_,
const google::protobuf::FieldDescriptor & field_descriptor_,
const ProtobufReaderOrWriter & reader_or_writer_)
: ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_)
@ -727,7 +727,7 @@ namespace
{
case FieldTypeId::TYPE_INT32:
{
write_function = [this](const std::string_view & str) { writeInt(parseFromStr<Int32>(str)); };
write_function = [this](std::string_view str) { writeInt(parseFromStr<Int32>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readInt(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); };
break;
@ -735,7 +735,7 @@ namespace
case FieldTypeId::TYPE_SINT32:
{
write_function = [this](const std::string_view & str) { writeSInt(parseFromStr<Int32>(str)); };
write_function = [this](std::string_view str) { writeSInt(parseFromStr<Int32>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readSInt(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); };
break;
@ -743,7 +743,7 @@ namespace
case FieldTypeId::TYPE_UINT32:
{
write_function = [this](const std::string_view & str) { writeUInt(parseFromStr<UInt32>(str)); };
write_function = [this](std::string_view str) { writeUInt(parseFromStr<UInt32>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readUInt(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_uint32()); };
break;
@ -751,7 +751,7 @@ namespace
case FieldTypeId::TYPE_INT64:
{
write_function = [this](const std::string_view & str) { writeInt(parseFromStr<Int64>(str)); };
write_function = [this](std::string_view str) { writeInt(parseFromStr<Int64>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readInt(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); };
break;
@ -759,7 +759,7 @@ namespace
case FieldTypeId::TYPE_SINT64:
{
write_function = [this](const std::string_view & str) { writeSInt(parseFromStr<Int64>(str)); };
write_function = [this](std::string_view str) { writeSInt(parseFromStr<Int64>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readSInt(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); };
break;
@ -767,7 +767,7 @@ namespace
case FieldTypeId::TYPE_UINT64:
{
write_function = [this](const std::string_view & str) { writeUInt(parseFromStr<UInt64>(str)); };
write_function = [this](std::string_view str) { writeUInt(parseFromStr<UInt64>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readUInt(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_uint64()); };
break;
@ -775,7 +775,7 @@ namespace
case FieldTypeId::TYPE_FIXED32:
{
write_function = [this](const std::string_view & str) { writeFixed<UInt32>(parseFromStr<UInt32>(str)); };
write_function = [this](std::string_view str) { writeFixed<UInt32>(parseFromStr<UInt32>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<UInt32>(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_uint32()); };
break;
@ -783,7 +783,7 @@ namespace
case FieldTypeId::TYPE_SFIXED32:
{
write_function = [this](const std::string_view & str) { writeFixed<Int32>(parseFromStr<Int32>(str)); };
write_function = [this](std::string_view str) { writeFixed<Int32>(parseFromStr<Int32>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Int32>(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); };
break;
@ -791,7 +791,7 @@ namespace
case FieldTypeId::TYPE_FIXED64:
{
write_function = [this](const std::string_view & str) { writeFixed<UInt64>(parseFromStr<UInt64>(str)); };
write_function = [this](std::string_view str) { writeFixed<UInt64>(parseFromStr<UInt64>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<UInt64>(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_uint64()); };
break;
@ -799,7 +799,7 @@ namespace
case FieldTypeId::TYPE_SFIXED64:
{
write_function = [this](const std::string_view & str) { writeFixed<Int64>(parseFromStr<Int64>(str)); };
write_function = [this](std::string_view str) { writeFixed<Int64>(parseFromStr<Int64>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Int64>(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); };
break;
@ -807,7 +807,7 @@ namespace
case FieldTypeId::TYPE_FLOAT:
{
write_function = [this](const std::string_view & str) { writeFixed<Float32>(parseFromStr<Float32>(str)); };
write_function = [this](std::string_view str) { writeFixed<Float32>(parseFromStr<Float32>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Float32>(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_float()); };
break;
@ -815,7 +815,7 @@ namespace
case FieldTypeId::TYPE_DOUBLE:
{
write_function = [this](const std::string_view & str) { writeFixed<Float64>(parseFromStr<Float64>(str)); };
write_function = [this](std::string_view str) { writeFixed<Float64>(parseFromStr<Float64>(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Float64>(), str); };
default_function = [this]() -> String { return toString(field_descriptor.default_value_double()); };
break;
@ -823,7 +823,7 @@ namespace
case FieldTypeId::TYPE_BOOL:
{
write_function = [this](const std::string_view & str)
write_function = [this](std::string_view str)
{
if (str == "true")
writeUInt(1);
@ -855,7 +855,7 @@ namespace
case FieldTypeId::TYPE_STRING:
case FieldTypeId::TYPE_BYTES:
{
write_function = [this](const std::string_view & str) { writeStr(str); };
write_function = [this](std::string_view str) { writeStr(str); };
read_function = [this](PaddedPODArray<UInt8> & str) { readStrAndAppend(str); };
default_function = [this]() -> String { return field_descriptor.default_value_string(); };
break;
@ -863,7 +863,7 @@ namespace
case FieldTypeId::TYPE_ENUM:
{
write_function = [this](const std::string_view & str) { writeInt(stringToProtobufEnumValue(str)); };
write_function = [this](std::string_view str) { writeInt(stringToProtobufEnumValue(str)); };
read_function = [this](PaddedPODArray<UInt8> & str) { protobufEnumValueToStringAppend(readInt(), str); };
default_function = [this]() -> String { return field_descriptor.default_value_enum()->name(); };
break;
@ -908,7 +908,7 @@ namespace
}
}
int stringToProtobufEnumValue(const std::string_view & str) const
int stringToProtobufEnumValue(std::string_view str) const
{
auto it = string_to_protobuf_enum_value_map.find(str);
if (it == string_to_protobuf_enum_value_map.end())
@ -932,7 +932,7 @@ namespace
const std::shared_ptr<const DataTypeFixedString> fixed_string_data_type;
const size_t n = 0;
std::function<void(const std::string_view &)> write_function;
std::function<void(std::string_view)> write_function;
std::function<void(PaddedPODArray<UInt8> &)> read_function;
std::function<String()> default_function;
std::unordered_map<std::string_view, int> string_to_protobuf_enum_value_map;
@ -953,7 +953,7 @@ namespace
using BaseClass = ProtobufSerializerNumber<NumberType>;
ProtobufSerializerEnum(
const std::string_view & column_name_,
std::string_view column_name_,
const std::shared_ptr<const EnumDataType> & enum_data_type_,
const FieldDescriptor & field_descriptor_,
const ProtobufReaderOrWriter & reader_or_writer_)
@ -1067,7 +1067,7 @@ namespace
protobuf_enum_value_to_enum_data_type_value_map.emplace(protobuf_enum_value, enum_data_type_value);
};
auto iless = [](const std::string_view & s1, const std::string_view & s2) { return ColumnNameWithProtobufFieldNameComparator::less(s1, s2); };
auto iless = [](std::string_view s1, std::string_view s2) { return ColumnNameWithProtobufFieldNameComparator::less(s1, s2); };
boost::container::flat_map<std::string_view, int, decltype(iless)> string_to_protobuf_enum_value_map;
typename decltype(string_to_protobuf_enum_value_map)::sequence_type string_to_protobuf_enum_value_seq;
for (int i : collections::range(enum_descriptor.value_count()))
@ -1133,9 +1133,9 @@ namespace
Int64 readInt() { return ProtobufSerializerSingleValue::readInt(); }
void writeInt(Int64 value) { ProtobufSerializerSingleValue::writeInt(value); }
void writeStr(const std::string_view & str) { ProtobufSerializerSingleValue::writeStr(str); }
void writeStr(std::string_view str) { ProtobufSerializerSingleValue::writeStr(str); }
void readStr(String & str) { ProtobufSerializerSingleValue::readStr(str); }
[[noreturn]] void cannotConvertValue(const std::string_view & src_value, const std::string_view & src_type_name, const std::string_view & dest_type_name) const { ProtobufSerializerSingleValue::cannotConvertValue(src_value, src_type_name, dest_type_name); }
[[noreturn]] void cannotConvertValue(std::string_view src_value, std::string_view src_type_name, std::string_view dest_type_name) const { ProtobufSerializerSingleValue::cannotConvertValue(src_value, src_type_name, dest_type_name); }
const std::shared_ptr<const EnumDataType> enum_data_type;
std::unordered_map<NumberType, int> enum_data_type_value_to_protobuf_enum_value_map;
@ -1152,7 +1152,7 @@ namespace
using ColumnType = ColumnDecimal<DecimalType>;
ProtobufSerializerDecimal(
const std::string_view & column_name_,
std::string_view column_name_,
const DataTypeDecimalBase<DecimalType> & decimal_data_type_,
const FieldDescriptor & field_descriptor_,
const ProtobufReaderOrWriter & reader_or_writer_)
@ -1412,7 +1412,7 @@ namespace
{
public:
ProtobufSerializerDate(
const std::string_view & column_name_,
std::string_view column_name_,
const FieldDescriptor & field_descriptor_,
const ProtobufReaderOrWriter & reader_or_writer_)
: ProtobufSerializerNumber<UInt16>(column_name_, field_descriptor_, reader_or_writer_)
@ -1490,7 +1490,7 @@ namespace
{
public:
ProtobufSerializerDateTime(
const std::string_view & column_name_,
std::string_view column_name_,
const DataTypeDateTime & type,
const FieldDescriptor & field_descriptor_,
const ProtobufReaderOrWriter & reader_or_writer_)
@ -1574,7 +1574,7 @@ namespace
{
public:
ProtobufSerializerUUID(
const std::string_view & column_name_,
std::string_view column_name_,
const google::protobuf::FieldDescriptor & field_descriptor_,
const ProtobufReaderOrWriter & reader_or_writer_)
: ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_)
@ -1654,7 +1654,7 @@ namespace
{
public:
ProtobufSerializerAggregateFunction(
const std::string_view & column_name_,
std::string_view column_name_,
const std::shared_ptr<const DataTypeAggregateFunction> & aggregate_function_data_type_,
const google::protobuf::FieldDescriptor & field_descriptor_,
const ProtobufReaderOrWriter & reader_or_writer_)
@ -2061,7 +2061,7 @@ namespace
{
public:
ProtobufSerializerTupleAsArray(
const std::string_view & column_name_,
std::string_view column_name_,
const std::shared_ptr<const DataTypeTuple> & tuple_data_type_,
const FieldDescriptor & field_descriptor_,
std::vector<std::unique_ptr<ProtobufSerializer>> element_serializers_)
@ -2833,7 +2833,7 @@ namespace
return field_names;
}
static bool columnNameEqualsToFieldName(const std::string_view & column_name, const FieldDescriptor & field_descriptor)
static bool columnNameEqualsToFieldName(std::string_view column_name, const FieldDescriptor & field_descriptor)
{
std::string_view suffix;
return columnNameStartsWithFieldName(column_name, field_descriptor, suffix) && suffix.empty();
@ -2844,7 +2844,7 @@ namespace
/// which doesn't match to the field's name.
/// The function requires that rest part of the column's name to be started with a dot '.' or underline '_',
/// but doesn't include those '.' or '_' characters into `suffix`.
static bool columnNameStartsWithFieldName(const std::string_view & column_name, const FieldDescriptor & field_descriptor, std::string_view & suffix)
static bool columnNameStartsWithFieldName(std::string_view column_name, const FieldDescriptor & field_descriptor, std::string_view & suffix)
{
size_t matching_length = 0;
const MessageDescriptor & containing_type = *field_descriptor.containing_type();
@ -2887,7 +2887,7 @@ namespace
/// for that case suffixes are also returned.
/// This is only the first filter, buildMessageSerializerImpl() does other checks after calling this function.
static bool findFieldsByColumnName(
const std::string_view & column_name,
std::string_view column_name,
const MessageDescriptor & message_descriptor,
std::vector<std::pair<const FieldDescriptor *, std::string_view /* suffix */>> & out_field_descriptors_with_suffixes,
bool google_wrappers_special_treatment)
@ -3030,7 +3030,7 @@ namespace
used_column_indices_sorted.reserve(num_columns);
size_t sequential_column_index = 0;
auto add_field_serializer = [&](const std::string_view & column_name_,
auto add_field_serializer = [&](std::string_view column_name_,
std::vector<size_t> && column_indices_,
const FieldDescriptor & field_descriptor_,
std::unique_ptr<ProtobufSerializer> field_serializer_)
@ -3243,7 +3243,7 @@ namespace
/// Builds a serializer for one-to-one match:
/// one column is serialized as one field in the protobuf message.
std::unique_ptr<ProtobufSerializer> buildFieldSerializer(
const std::string_view & column_name,
std::string_view column_name,
const DataTypePtr & data_type,
const FieldDescriptor & field_descriptor,
bool allow_repeat,
@ -3395,7 +3395,7 @@ namespace
}
}
[[noreturn]] static void throwFieldNotRepeated(const FieldDescriptor & field_descriptor, const std::string_view & column_name)
[[noreturn]] static void throwFieldNotRepeated(const FieldDescriptor & field_descriptor, std::string_view column_name)
{
if (!field_descriptor.is_repeated())
throw Exception(

2
src/Formats/ProtobufWriter.cpp
View File

@ -196,7 +196,7 @@ template void ProtobufWriter::writeFixed<UInt64>(int field_number, UInt64 value)
template void ProtobufWriter::writeFixed<Float32>(int field_number, Float32 value);
template void ProtobufWriter::writeFixed<Float64>(int field_number, Float64 value);
void ProtobufWriter::writeString(int field_number, const std::string_view & str)
void ProtobufWriter::writeString(int field_number, std::string_view str)
{
size_t length = str.length();
size_t old_size = buffer.size();

2
src/Formats/ProtobufWriter.h
View File

@ -30,7 +30,7 @@ public:
void writeSInt(int field_number, Int64 value);
template <typename T>
void writeFixed(int field_number, T value);
void writeString(int field_number, const std::string_view & str);
void writeString(int field_number, std::string_view str);
void startRepeatedPack();
void endRepeatedPack(int field_number, bool skip_if_empty);

30
src/Functions/FunctionsJSON.cpp
View File

@ -304,7 +304,7 @@ private:
/// Performs moves of types MoveType::Key and MoveType::ConstKey.
template <typename JSONParser>
static bool moveToElementByKey(typename JSONParser::Element & element, const std::string_view & key)
static bool moveToElementByKey(typename JSONParser::Element & element, std::string_view key)
{
if (!element.isObject())
return false;
@ -504,7 +504,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view &)
static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view)
{
ColumnVector<UInt8> & col_vec = assert_cast<ColumnVector<UInt8> &>(dest);
col_vec.insertValue(1);
@ -532,7 +532,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName &) { return 0; }
static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view &)
static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view)
{
/// This function is called only if JSON is valid.
/// If JSON isn't valid then `FunctionJSON::Executor::run()` adds default value (=zero) to `dest` without calling this function.
@ -556,7 +556,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
size_t size;
if (element.isArray())
@ -586,7 +586,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view & last_key)
static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view last_key)
{
if (last_key.empty())
return false;
@ -620,7 +620,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
UInt8 type;
if (element.isInt64())
@ -662,7 +662,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
NumberType value;
@ -737,7 +737,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
if (!element.isBool())
return false;
@ -764,7 +764,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
if (element.isNull())
return false;
@ -1164,7 +1164,7 @@ public:
extract_tree = JSONExtractTree<JSONParser>::build(function_name, result_type);
}
bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
return extract_tree->insertResultToColumn(dest, element);
}
@ -1207,7 +1207,7 @@ public:
extract_tree = JSONExtractTree<JSONParser>::build(function_name, value_type);
}
bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
if (!element.isObject())
return false;
@ -1251,7 +1251,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
ColumnString & col_str = assert_cast<ColumnString &>(dest);
auto & chars = col_str.getChars();
@ -1355,7 +1355,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
if (!element.isArray())
return false;
@ -1387,7 +1387,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
if (!element.isObject())
return false;
@ -1423,7 +1423,7 @@ public:
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
{
if (!element.isObject())
return false;

7
src/IO/ReadBufferFromAzureBlobStorage.cpp
View File

@ -5,6 +5,7 @@
#include <IO/ReadBufferFromAzureBlobStorage.h>
#include <IO/ReadBufferFromString.h>
#include <Common/logger_useful.h>
#include <Common/Throttler.h>
#include <base/sleep.h>
@ -26,7 +27,6 @@ ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage(
const ReadSettings & read_settings_,
size_t max_single_read_retries_,
size_t max_single_download_retries_,
size_t tmp_buffer_size_,
bool use_external_buffer_,
size_t read_until_position_)
: ReadBufferFromFileBase(read_settings_.remote_fs_buffer_size, nullptr, 0)
@ -34,7 +34,8 @@ ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage(
, path(path_)
, max_single_read_retries(max_single_read_retries_)
, max_single_download_retries(max_single_download_retries_)
, tmp_buffer_size(tmp_buffer_size_)
, read_settings(read_settings_)
, tmp_buffer_size(read_settings.remote_fs_buffer_size)
, use_external_buffer(use_external_buffer_)
, read_until_position(read_until_position_)
{
@ -76,6 +77,8 @@ bool ReadBufferFromAzureBlobStorage::nextImpl()
try
{
bytes_read = data_stream->ReadToCount(reinterpret_cast<uint8_t *>(data_ptr), to_read_bytes);
if (read_settings.remote_throttler)
read_settings.remote_throttler->add(bytes_read);
break;
}
catch (const Azure::Storage::StorageException & e)

2
src/IO/ReadBufferFromAzureBlobStorage.h
View File

@ -23,7 +23,6 @@ public:
const ReadSettings & read_settings_,
size_t max_single_read_retries_,
size_t max_single_download_retries_,
size_t tmp_buffer_size_,
bool use_external_buffer_ = false,
size_t read_until_position_ = 0);
@ -48,6 +47,7 @@ private:
const String path;
size_t max_single_read_retries;
size_t max_single_download_retries;
ReadSettings read_settings;
std::vector<char> tmp_buffer;
size_t tmp_buffer_size;
bool use_external_buffer;

6
src/IO/ReadBufferFromS3.cpp
View File

@ -5,12 +5,13 @@
#include <IO/ReadBufferFromIStream.h>
#include <IO/ReadBufferFromS3.h>
#include <Common/Stopwatch.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <Common/Stopwatch.h>
#include <Common/Throttler.h>
#include <Common/logger_useful.h>
#include <base/sleep.h>
@ -164,6 +165,8 @@ bool ReadBufferFromS3::nextImpl()
ProfileEvents::increment(ProfileEvents::ReadBufferFromS3Bytes, working_buffer.size());
offset += working_buffer.size();
if (read_settings.remote_throttler)
read_settings.remote_throttler->add(working_buffer.size());
return true;
}
@ -299,7 +302,6 @@ std::unique_ptr<ReadBuffer> ReadBufferFromS3::initialize()
if (outcome.IsSuccess())
{
read_result = outcome.GetResultWithOwnership();
size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size;
return std::make_unique<ReadBufferFromIStream>(read_result.GetBody(), buffer_size);
}

4
src/IO/ReadHelpers.h
View File

@ -836,7 +836,7 @@ template <typename T>
inline T parse(const char * data, size_t size);
template <typename T>
inline T parseFromString(const std::string_view & str)
inline T parseFromString(std::string_view str)
{
return parse<T>(str.data(), str.size());
}
@ -1338,7 +1338,7 @@ inline T parseWithSizeSuffix(const char * data, size_t size)
}
template <typename T>
inline T parseWithSizeSuffix(const std::string_view & s)
inline T parseWithSizeSuffix(std::string_view s)
{
return parseWithSizeSuffix<T>(s.data(), s.size());
}

4
src/IO/ReadSettings.h
View File

@ -4,6 +4,7 @@
#include <string>
#include <Core/Defines.h>
#include <Common/FileCache_fwd.h>
#include <Common/Throttler_fwd.h>
namespace DB
{
@ -89,6 +90,9 @@ struct ReadSettings
FileCachePtr remote_fs_cache;
/// Bandwidth throttler to use during reading
ThrottlerPtr remote_throttler;
size_t http_max_tries = 1;
size_t http_retry_initial_backoff_ms = 100;
size_t http_retry_max_backoff_ms = 1600;

6
src/IO/WriteBufferFromAzureBlobStorage.cpp
View File

@ -5,6 +5,7 @@
#include <IO/WriteBufferFromAzureBlobStorage.h>
#include <Common/getRandomASCIIString.h>
#include <Common/logger_useful.h>
#include <Common/Throttler.h>
namespace DB
@ -15,11 +16,13 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage(
const String & blob_path_,
size_t max_single_part_upload_size_,
size_t buf_size_,
const WriteSettings & write_settings_,
std::optional<std::map<std::string, std::string>> attributes_)
: BufferWithOwnMemory<WriteBuffer>(buf_size_, nullptr, 0)
, blob_container_client(blob_container_client_)
, max_single_part_upload_size(max_single_part_upload_size_)
, blob_path(blob_path_)
, write_settings(write_settings_)
, attributes(attributes_)
{
}
@ -84,6 +87,9 @@ void WriteBufferFromAzureBlobStorage::nextImpl()
}
block_blob_client.CommitBlockList(block_ids);
if (write_settings.remote_throttler)
write_settings.remote_throttler->add(read);
}
}

3
src/IO/WriteBufferFromAzureBlobStorage.h
View File

@ -8,6 +8,7 @@
#include <IO/BufferWithOwnMemory.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteSettings.h>
#include <azure/storage/blobs.hpp>
#include <azure/core/io/body_stream.hpp>
@ -24,6 +25,7 @@ public:
const String & blob_path_,
size_t max_single_part_upload_size_,
size_t buf_size_,
const WriteSettings & write_settings_,
std::optional<std::map<std::string, std::string>> attributes_ = {});
~WriteBufferFromAzureBlobStorage() override;
@ -36,6 +38,7 @@ private:
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> blob_container_client;
size_t max_single_part_upload_size;
const String blob_path;
WriteSettings write_settings;
std::optional<std::map<std::string, std::string>> attributes;
};

7
src/IO/WriteBufferFromS3.cpp
View File

@ -4,6 +4,7 @@
#include <Common/logger_useful.h>
#include <Common/IFileCache.h>
#include <Common/Throttler.h>
#include <IO/WriteBufferFromS3.h>
#include <IO/WriteHelpers.h>
@ -61,6 +62,7 @@ WriteBufferFromS3::WriteBufferFromS3(
std::optional<std::map<String, String>> object_metadata_,
size_t buffer_size_,
ScheduleFunc schedule_,
const WriteSettings & write_settings_,
FileCachePtr cache_)
: BufferWithOwnMemory<WriteBuffer>(buffer_size_, nullptr, 0)
, bucket(bucket_)
@ -70,6 +72,7 @@ WriteBufferFromS3::WriteBufferFromS3(
, s3_settings(s3_settings_)
, object_metadata(std::move(object_metadata_))
, schedule(std::move(schedule_))
, write_settings(write_settings_)
, cache(cache_)
{
allocateBuffer();
@ -121,8 +124,9 @@ void WriteBufferFromS3::nextImpl()
}
ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, offset());
last_part_size += offset();
if (write_settings.remote_throttler)
write_settings.remote_throttler->add(offset());
/// Data size exceeds singlepart upload threshold, need to use multipart upload.
if (multipart_upload_id.empty() && last_part_size > s3_settings.max_single_part_upload_size)
@ -462,7 +466,6 @@ void WriteBufferFromS3::processPutRequest(PutObjectTask & task)
{
auto outcome = client_ptr->PutObject(task.req);
bool with_pool = static_cast<bool>(schedule);
if (outcome.IsSuccess())
LOG_TRACE(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool);
else

4
src/IO/WriteBufferFromS3.h
View File

@ -16,6 +16,7 @@
#include <IO/BufferWithOwnMemory.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteSettings.h>
#include <Storages/StorageS3Settings.h>
#include <aws/core/utils/memory/stl/AWSStringStream.h>
@ -55,6 +56,7 @@ public:
std::optional<std::map<String, String>> object_metadata_ = std::nullopt,
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE,
ScheduleFunc schedule_ = {},
const WriteSettings & write_settings_ = {},
FileCachePtr cache_ = nullptr);
~WriteBufferFromS3() override;
@ -119,6 +121,8 @@ private:
Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3");
WriteSettings write_settings;
FileCachePtr cache;
size_t current_download_offset = 0;
std::optional<FileSegmentsHolder> file_segments_holder;

16
src/IO/WriteHelpers.h
View File

@ -113,7 +113,7 @@ inline void writeStringBinary(const char * s, WriteBuffer & buf)
writeStringBinary(StringRef{s}, buf);
}
inline void writeStringBinary(const std::string_view & s, WriteBuffer & buf)
inline void writeStringBinary(std::string_view s, WriteBuffer & buf)
{
writeStringBinary(StringRef{s}, buf);
}
@ -365,7 +365,7 @@ inline void writeJSONString(const StringRef & s, WriteBuffer & buf, const Format
writeJSONString(s.data, s.data + s.size, buf, settings);
}
inline void writeJSONString(const std::string_view & s, WriteBuffer & buf, const FormatSettings & settings)
inline void writeJSONString(std::string_view s, WriteBuffer & buf, const FormatSettings & settings)
{
writeJSONString(StringRef{s}, buf, settings);
}
@ -440,7 +440,7 @@ inline void writeEscapedString(const StringRef & ref, WriteBuffer & buf)
writeEscapedString(ref.data, ref.size, buf);
}
inline void writeEscapedString(const std::string_view & ref, WriteBuffer & buf)
inline void writeEscapedString(std::string_view ref, WriteBuffer & buf)
{
writeEscapedString(ref.data(), ref.size(), buf);
}
@ -478,7 +478,7 @@ inline void writeQuotedString(const StringRef & ref, WriteBuffer & buf)
writeAnyQuotedString<'\''>(ref, buf);
}
inline void writeQuotedString(const std::string_view & ref, WriteBuffer & buf)
inline void writeQuotedString(std::string_view ref, WriteBuffer & buf)
{
writeAnyQuotedString<'\''>(ref.data(), ref.data() + ref.size(), buf);
}
@ -493,7 +493,7 @@ inline void writeDoubleQuotedString(const StringRef & s, WriteBuffer & buf)
writeAnyQuotedString<'"'>(s, buf);
}
inline void writeDoubleQuotedString(const std::string_view & s, WriteBuffer & buf)
inline void writeDoubleQuotedString(std::string_view s, WriteBuffer & buf)
{
writeAnyQuotedString<'"'>(s.data(), s.data() + s.size(), buf);
}
@ -891,7 +891,7 @@ inline void writeBinary(const T & x, WriteBuffer & buf) { writePODBinary(x, buf)
inline void writeBinary(const String & x, WriteBuffer & buf) { writeStringBinary(x, buf); }
inline void writeBinary(const StringRef & x, WriteBuffer & buf) { writeStringBinary(x, buf); }
inline void writeBinary(const std::string_view & x, WriteBuffer & buf) { writeStringBinary(x, buf); }
inline void writeBinary(std::string_view x, WriteBuffer & buf) { writeStringBinary(x, buf); }
inline void writeBinary(const Decimal32 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Decimal64 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Decimal128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
@ -1015,7 +1015,7 @@ inline void writeQuoted(const T & x, WriteBuffer & buf) { writeText(x, buf); }
inline void writeQuoted(const String & x, WriteBuffer & buf) { writeQuotedString(x, buf); }
inline void writeQuoted(const std::string_view & x, WriteBuffer & buf) { writeQuotedString(x, buf); }
inline void writeQuoted(std::string_view x, WriteBuffer & buf) { writeQuotedString(x, buf); }
inline void writeQuoted(const StringRef & x, WriteBuffer & buf) { writeQuotedString(x, buf); }
@ -1048,7 +1048,7 @@ inline void writeDoubleQuoted(const T & x, WriteBuffer & buf) { writeText(x, buf
inline void writeDoubleQuoted(const String & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); }
inline void writeDoubleQuoted(const std::string_view & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); }
inline void writeDoubleQuoted(std::string_view x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); }
inline void writeDoubleQuoted(const StringRef & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); }

5
src/IO/WriteSettings.h
View File

@ -1,5 +1,7 @@
#pragma once
#include <Common/Throttler_fwd.h>
namespace DB
{
@ -7,6 +9,9 @@ namespace DB
struct WriteSettings
{
bool enable_filesystem_cache_on_write_operations = false;
/// Bandwidth throttler to use during writing
ThrottlerPtr remote_throttler;
};
}

42
src/Interpreters/Context.cpp
View File

@ -228,8 +228,10 @@ struct ContextSharedPart
mutable std::unique_ptr<BackgroundSchedulePool> distributed_schedule_pool; /// A thread pool that can run different jobs in background (used for distributed sends)
mutable std::unique_ptr<BackgroundSchedulePool> message_broker_schedule_pool; /// A thread pool that can run different jobs in background (used for message brokers, like RabbitMQ and Kafka)
mutable ThrottlerPtr replicated_fetches_throttler; /// A server-wide throttler for replicated fetches
mutable ThrottlerPtr replicated_sends_throttler; /// A server-wide throttler for replicated sends
mutable ThrottlerPtr replicated_fetches_throttler; /// A server-wide throttler for replicated fetches
mutable ThrottlerPtr replicated_sends_throttler; /// A server-wide throttler for replicated sends
mutable ThrottlerPtr remote_read_throttler; /// A server-wide throttler for remote IO reads
mutable ThrottlerPtr remote_write_throttler; /// A server-wide throttler for remote IO writes
MultiVersion<Macros> macros; /// Substitutions extracted from config.
std::unique_ptr<DDLWorker> ddl_worker; /// Process ddl commands from zk.
@ -854,13 +856,13 @@ void Context::checkAccessImpl(const Args &... args) const
}
void Context::checkAccess(const AccessFlags & flags) const { return checkAccessImpl(flags); }
void Context::checkAccess(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl(flags, database); }
void Context::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl(flags, database, table); }
void Context::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl(flags, database, table, column); }
void Context::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl(flags, database, table, columns); }
void Context::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl(flags, database, table, columns); }
void Context::checkAccess(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl(flags, database); }
void Context::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl(flags, database, table); }
void Context::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl(flags, database, table, column); }
void Context::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl(flags, database, table, columns); }
void Context::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl(flags, database, table, columns); }
void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id) const { checkAccessImpl(flags, table_id.getDatabaseName(), table_id.getTableName()); }
void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id, const std::string_view & column) const { checkAccessImpl(flags, table_id.getDatabaseName(), table_id.getTableName(), column); }
void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id, std::string_view column) const { checkAccessImpl(flags, table_id.getDatabaseName(), table_id.getTableName(), column); }
void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id, const std::vector<std::string_view> & columns) const { checkAccessImpl(flags, table_id.getDatabaseName(), table_id.getTableName(), columns); }
void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id, const Strings & columns) const { checkAccessImpl(flags, table_id.getDatabaseName(), table_id.getTableName(), columns); }
void Context::checkAccess(const AccessRightsElement & element) const { return checkAccessImpl(element); }
@ -1930,6 +1932,26 @@ ThrottlerPtr Context::getReplicatedSendsThrottler() const
return shared->replicated_sends_throttler;
}
ThrottlerPtr Context::getRemoteReadThrottler() const
{
auto lock = getLock();
if (!shared->remote_read_throttler)
shared->remote_read_throttler = std::make_shared<Throttler>(
settings.max_remote_read_network_bandwidth_for_server);
return shared->remote_read_throttler;
}
ThrottlerPtr Context::getRemoteWriteThrottler() const
{
auto lock = getLock();
if (!shared->remote_write_throttler)
shared->remote_write_throttler = std::make_shared<Throttler>(
settings.max_remote_write_network_bandwidth_for_server);
return shared->remote_write_throttler;
}
bool Context::hasDistributedDDL() const
{
return getConfigRef().has("distributed_ddl");
@ -3436,6 +3458,8 @@ ReadSettings Context::getReadSettings() const
res.mmap_threshold = settings.min_bytes_to_use_mmap_io;
res.priority = settings.read_priority;
res.remote_throttler = getRemoteReadThrottler();
res.http_max_tries = settings.http_max_tries;
res.http_retry_initial_backoff_ms = settings.http_retry_initial_backoff_ms;
res.http_retry_max_backoff_ms = settings.http_retry_max_backoff_ms;
@ -3452,6 +3476,8 @@ WriteSettings Context::getWriteSettings() const
res.enable_filesystem_cache_on_write_operations = settings.enable_filesystem_cache_on_write_operations;
res.remote_throttler = getRemoteWriteThrottler();
return res;
}

14
src/Interpreters/Context.h
View File

@ -474,13 +474,13 @@ public:
/// Checks access rights.
/// Empty database means the current database.
void checkAccess(const AccessFlags & flags) const;
void checkAccess(const AccessFlags & flags, const std::string_view & database) const;
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
void checkAccess(const AccessFlags & flags, std::string_view database) const;
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const;
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
void checkAccess(const AccessFlags & flags, const StorageID & table_id) const;
void checkAccess(const AccessFlags & flags, const StorageID & table_id, const std::string_view & column) const;
void checkAccess(const AccessFlags & flags, const StorageID & table_id, std::string_view column) const;
void checkAccess(const AccessFlags & flags, const StorageID & table_id, const std::vector<std::string_view> & columns) const;
void checkAccess(const AccessFlags & flags, const StorageID & table_id, const Strings & columns) const;
void checkAccess(const AccessRightsElement & element) const;
@ -820,6 +820,8 @@ public:
ThrottlerPtr getReplicatedFetchesThrottler() const;
ThrottlerPtr getReplicatedSendsThrottler() const;
ThrottlerPtr getRemoteReadThrottler() const;
ThrottlerPtr getRemoteWriteThrottler() const;
/// Has distributed_ddl configuration or not.
bool hasDistributedDDL() const;

11
src/Interpreters/DatabaseCatalog.cpp
View File

@ -234,7 +234,7 @@ void DatabaseCatalog::shutdownImpl()
view_dependencies.clear();
}
bool DatabaseCatalog::isPredefinedDatabase(const std::string_view & database_name)
bool DatabaseCatalog::isPredefinedDatabase(std::string_view database_name)
{
return database_name == TEMPORARY_DATABASE || database_name == SYSTEM_DATABASE || database_name == INFORMATION_SCHEMA
|| database_name == INFORMATION_SCHEMA_UPPERCASE;
@ -403,10 +403,11 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas
std::lock_guard lock{databases_mutex};
assertDatabaseDoesntExistUnlocked(database_name);
databases.emplace(database_name, database);
NOEXCEPT_SCOPE;
UUID db_uuid = database->getUUID();
if (db_uuid != UUIDHelpers::Nil)
addUUIDMapping(db_uuid, database, nullptr);
NOEXCEPT_SCOPE({
UUID db_uuid = database->getUUID();
if (db_uuid != UUIDHelpers::Nil)
addUUIDMapping(db_uuid, database, nullptr);
});
}

2
src/Interpreters/DatabaseCatalog.h
View File

@ -131,7 +131,7 @@ public:
static constexpr const char * INFORMATION_SCHEMA_UPPERCASE = "INFORMATION_SCHEMA";
/// Returns true if a passed name is one of the predefined databases' names.
static bool isPredefinedDatabase(const std::string_view & database_name);
static bool isPredefinedDatabase(std::string_view database_name);
static DatabaseCatalog & init(ContextMutablePtr global_context_);
static DatabaseCatalog & instance();

36
src/Interpreters/ExpressionAnalyzer.cpp
View File

@ -1253,7 +1253,7 @@ JoinPtr SelectQueryExpressionAnalyzer::makeJoin(
}
ActionsDAGPtr SelectQueryExpressionAnalyzer::appendPrewhere(
ExpressionActionsChain & chain, bool only_types, const Names & additional_required_columns)
ExpressionActionsChain & chain, bool only_types)
{
const auto * select_query = getSelectQuery();
if (!select_query->prewhere())
@ -1290,14 +1290,6 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendPrewhere(
NameSet required_source_columns(required_columns.begin(), required_columns.end());
required_source_columns.insert(first_action_names.begin(), first_action_names.end());
/// Add required columns to required output in order not to remove them after prewhere execution.
/// TODO: add sampling and final execution to common chain.
for (const auto & column : additional_required_columns)
{
if (required_source_columns.contains(column))
step.addRequiredOutput(column);
}
auto names = step.actions()->getNames();
NameSet name_set(names.begin(), names.end());
@ -1844,12 +1836,28 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
const Settings & settings = context->getSettingsRef();
const ConstStoragePtr & storage = query_analyzer.storage();
Names additional_required_columns_after_prewhere;
ssize_t prewhere_step_num = -1;
ssize_t where_step_num = -1;
ssize_t having_step_num = -1;
auto finalize_chain = [&](ExpressionActionsChain & chain)
{
if (prewhere_step_num >= 0)
{
ExpressionActionsChain::Step & step = *chain.steps.at(prewhere_step_num);
auto required_columns = prewhere_info->prewhere_actions->getRequiredColumnsNames();
NameSet required_source_columns(required_columns.begin(), required_columns.end());
/// Add required columns to required output in order not to remove them after prewhere execution.
/// TODO: add sampling and final execution to common chain.
for (const auto & column : additional_required_columns_after_prewhere)
{
if (required_source_columns.contains(column))
step.addRequiredOutput(column);
}
}
chain.finalize();
finalize(chain, prewhere_step_num, where_step_num, having_step_num, query);
@ -1859,7 +1867,6 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
{
ExpressionActionsChain chain(context);
Names additional_required_columns_after_prewhere;
if (storage && (query.sampleSize() || settings.parallel_replicas_count > 1))
{
@ -1881,7 +1888,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
filter_info->do_remove_column = true;
}
if (auto actions = query_analyzer.appendPrewhere(chain, !first_stage, additional_required_columns_after_prewhere))
if (auto actions = query_analyzer.appendPrewhere(chain, !first_stage))
{
/// Prewhere is always the first one.
prewhere_step_num = 0;
@ -1976,6 +1983,13 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
&& !query.final()
&& join_allow_read_in_order;
if (storage && optimize_read_in_order)
{
Names columns_for_sorting_key = metadata_snapshot->getColumnsRequiredForSortingKey();
additional_required_columns_after_prewhere.insert(additional_required_columns_after_prewhere.end(),
columns_for_sorting_key.begin(), columns_for_sorting_key.end());
}
/// If there is aggregation, we execute expressions in SELECT and ORDER BY on the initiating server, otherwise on the source servers.
query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage));

2
src/Interpreters/ExpressionAnalyzer.h
View File

@ -403,7 +403,7 @@ private:
/// remove_filter is set in ExpressionActionsChain::finalize();
/// Columns in `additional_required_columns` will not be removed (they can be used for e.g. sampling or FINAL modifier).
ActionsDAGPtr appendPrewhere(ExpressionActionsChain & chain, bool only_types, const Names & additional_required_columns);
ActionsDAGPtr appendPrewhere(ExpressionActionsChain & chain, bool only_types);
bool appendWhere(ExpressionActionsChain & chain, bool only_types);
bool appendGroupBy(ExpressionActionsChain & chain, bool only_types, bool optimize_aggregation_in_order, ManyExpressionActions &);
void appendAggregateFunctionsArguments(ExpressionActionsChain & chain, bool only_types);

11
src/Interpreters/MergeTreeTransaction.cpp
View File

@ -149,11 +149,12 @@ void MergeTreeTransaction::removeOldPart(const StoragePtr & storage, const DataP
checkIsNotCancelled();
part_to_remove->version.lockRemovalTID(tid, context);
NOEXCEPT_SCOPE;
storages.insert(storage);
if (maybe_lock)
table_read_locks_for_ordinary_db.emplace_back(std::move(maybe_lock));
removing_parts.push_back(part_to_remove);
NOEXCEPT_SCOPE({
storages.insert(storage);
if (maybe_lock)
table_read_locks_for_ordinary_db.emplace_back(std::move(maybe_lock));
removing_parts.push_back(part_to_remove);
});
}
part_to_remove->appendRemovalTIDToVersionMetadata();

2
src/Interpreters/ThreadStatusExt.cpp
View File

@ -343,7 +343,7 @@ void ThreadStatus::finalizeQueryProfiler()
void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits)
{
NOEXCEPT_SCOPE;
LockMemoryExceptionInThread lock_memory_tracker(VariableContext::Global);
if (exit_if_already_detached && thread_state == ThreadState::DetachedFromQuery)
{

15
src/Interpreters/TransactionLog.cpp
View File

@ -140,8 +140,7 @@ void TransactionLog::loadEntries(Strings::const_iterator beg, Strings::const_ite
}
futures.clear();
NOEXCEPT_SCOPE_STRICT;
{
NOEXCEPT_SCOPE_STRICT({
std::lock_guard lock{mutex};
for (const auto & entry : loaded)
{
@ -151,7 +150,8 @@ void TransactionLog::loadEntries(Strings::const_iterator beg, Strings::const_ite
tid_to_csn.emplace(entry.first, entry.second);
}
last_loaded_entry = last_entry;
}
});
{
std::lock_guard lock{running_list_mutex};
latest_snapshot = loaded.back().second.csn;
@ -445,10 +445,11 @@ CSN TransactionLog::commitTransaction(const MergeTreeTransactionPtr & txn, bool
/// Do not allow exceptions between commit point and the and of transaction finalization
/// (otherwise it may stuck in COMMITTING state holding snapshot).
NOEXCEPT_SCOPE_STRICT;
/// FIXME Transactions: Sequential node numbers in ZooKeeper are Int32, but 31 bit is not enough for production use
/// (overflow is possible in a several weeks/months of active usage)
allocated_csn = deserializeCSN(csn_path_created.substr(zookeeper_path_log.size() + 1));
NOEXCEPT_SCOPE_STRICT({
/// FIXME Transactions: Sequential node numbers in ZooKeeper are Int32, but 31 bit is not enough for production use
/// (overflow is possible in a several weeks/months of active usage)
allocated_csn = deserializeCSN(csn_path_created.substr(zookeeper_path_log.size() + 1));
});
}
return finalizeCommittedTransaction(txn.get(), allocated_csn, state_guard);

2
src/Interpreters/TranslateQualifiedNamesVisitor.cpp
View File

@ -31,7 +31,7 @@ namespace ErrorCodes
extern const int UNSUPPORTED_JOIN_KEYS;
extern const int LOGICAL_ERROR;
}
bool TranslateQualifiedNamesMatcher::Data::matchColumnName(const std::string_view & name, const String & column_name, DataTypePtr column_type)
bool TranslateQualifiedNamesMatcher::Data::matchColumnName(std::string_view name, const String & column_name, DataTypePtr column_type)
{
if (name.size() < column_name.size())
return false;

2
src/Interpreters/TranslateQualifiedNamesVisitor.h
View File

@ -39,7 +39,7 @@ public:
bool hasTable() const { return !tables.empty(); }
bool processAsterisks() const { return hasTable() && has_columns; }
bool unknownColumn(size_t table_pos, const ASTIdentifier & identifier) const;
static bool matchColumnName(const std::string_view & name, const String & column_name, DataTypePtr column_type);
static bool matchColumnName(std::string_view name, const String & column_name, DataTypePtr column_type);
};
static void visit(ASTPtr & ast, Data & data);

2
src/Interpreters/UserDefinedSQLObjectsLoader.cpp
View File

@ -43,7 +43,7 @@ UserDefinedSQLObjectsLoader::UserDefinedSQLObjectsLoader()
: log(&Poco::Logger::get("UserDefinedSQLObjectsLoader"))
{}
void UserDefinedSQLObjectsLoader::loadUserDefinedObject(ContextPtr context, UserDefinedSQLObjectType object_type, const std::string_view & name, const String & path)
void UserDefinedSQLObjectsLoader::loadUserDefinedObject(ContextPtr context, UserDefinedSQLObjectType object_type, std::string_view name, const String & path)
{
auto name_ref = StringRef(name.data(), name.size());
LOG_DEBUG(log, "Loading user defined object {} from file {}", backQuote(name_ref), path);

2
src/Interpreters/UserDefinedSQLObjectsLoader.h
View File

@ -29,7 +29,7 @@ public:
private:
void loadUserDefinedObject(ContextPtr context, UserDefinedSQLObjectType object_type, const std::string_view & object_name, const String & file_path);
void loadUserDefinedObject(ContextPtr context, UserDefinedSQLObjectType object_type, std::string_view object_name, const String & file_path);
Poco::Logger * log;
bool enable_persistence = true;
};

2
src/Interpreters/executeQuery.cpp
View File

@ -876,7 +876,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
{
auto progress_out = process_list_elem->getProgressOut();
elem.result_rows = progress_out.written_rows;
elem.result_bytes = progress_out.written_rows;
elem.result_bytes = progress_out.written_bytes;
}
if (elem.read_rows != 0)

4
src/Parsers/Access/ParserCreateRowPolicyQuery.cpp
View File

@ -75,7 +75,7 @@ namespace
{
for (auto filter_type : collections::range(RowPolicyFilterType::MAX))
{
const std::string_view & command = RowPolicyFilterTypeInfo::get(filter_type).command;
std::string_view command = RowPolicyFilterTypeInfo::get(filter_type).command;
commands.emplace(command);
}
}
@ -96,7 +96,7 @@ namespace
for (auto filter_type : collections::range(RowPolicyFilterType::MAX))
{
const std::string_view & command = RowPolicyFilterTypeInfo::get(filter_type).command;
std::string_view command = RowPolicyFilterTypeInfo::get(filter_type).command;
if (ParserKeyword{command.data()}.ignore(pos, expected))
{
res_commands.emplace(command);

9
src/Parsers/ExpressionListParsers.h
View File

@ -9,6 +9,11 @@
#include <Parsers/SelectUnionMode.h>
#include <Common/IntervalKind.h>
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wc99-extensions"
#endif
namespace DB
{
@ -564,3 +569,7 @@ protected:
};
}
#ifdef __clang__
#pragma clang diagnostic pop
#endif

13
src/Storages/HDFS/ReadBufferFromHDFS.cpp
View File

@ -2,6 +2,7 @@
#if USE_HDFS
#include <Storages/HDFS/HDFSCommon.h>
#include <Common/Throttler.h>
#include <hdfs/hdfs.h>
#include <mutex>
@ -24,13 +25,13 @@ ReadBufferFromHDFS::~ReadBufferFromHDFS() = default;
struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory<SeekableReadBuffer>
{
String hdfs_uri;
String hdfs_file_path;
hdfsFile fin;
HDFSBuilderWrapper builder;
HDFSFSPtr fs;
ReadSettings read_settings;
off_t file_offset = 0;
off_t read_until_position = 0;
@ -39,11 +40,13 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory<S
const std::string & hdfs_uri_,
const std::string & hdfs_file_path_,
const Poco::Util::AbstractConfiguration & config_,
size_t buf_size_, size_t read_until_position_)
: BufferWithOwnMemory<SeekableReadBuffer>(buf_size_)
const ReadSettings & read_settings_,
size_t read_until_position_)
: BufferWithOwnMemory<SeekableReadBuffer>(read_settings_.remote_fs_buffer_size)
, hdfs_uri(hdfs_uri_)
, hdfs_file_path(hdfs_file_path_)
, builder(createHDFSBuilder(hdfs_uri_, config_))
, read_settings(read_settings_)
, read_until_position(read_until_position_)
{
fs = createHDFSFS(builder.get());
@ -97,6 +100,8 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory<S
working_buffer = internal_buffer;
working_buffer.resize(bytes_read);
file_offset += bytes_read;
if (read_settings.remote_throttler)
read_settings.remote_throttler->add(bytes_read);
return true;
}
@ -130,7 +135,7 @@ ReadBufferFromHDFS::ReadBufferFromHDFS(
size_t read_until_position_)
: ReadBufferFromFileBase(read_settings_.remote_fs_buffer_size, nullptr, 0)
, impl(std::make_unique<ReadBufferFromHDFSImpl>(
hdfs_uri_, hdfs_file_path_, config_, read_settings_.remote_fs_buffer_size, read_until_position_))
hdfs_uri_, hdfs_file_path_, config_, read_settings_, read_until_position_))
{
}

9
src/Storages/HDFS/StorageHDFS.cpp
View File

@ -16,6 +16,7 @@
#include <IO/WriteHelpers.h>
#include <IO/CompressionMethod.h>
#include <IO/WriteSettings.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Interpreters/ExpressionAnalyzer.h>
@ -412,7 +413,13 @@ public:
const CompressionMethod compression_method)
: SinkToStorage(sample_block)
{
write_buf = wrapWriteBufferWithCompressionMethod(std::make_unique<WriteBufferFromHDFS>(uri, context->getGlobalContext()->getConfigRef(), context->getSettingsRef().hdfs_replication), compression_method, 3);
write_buf = wrapWriteBufferWithCompressionMethod(
std::make_unique<WriteBufferFromHDFS>(
uri,
context->getGlobalContext()->getConfigRef(),
context->getSettingsRef().hdfs_replication,
context->getWriteSettings()),
compression_method, 3);
writer = FormatFactory::instance().getOutputFormatParallelIfPossible(format, *write_buf, sample_block, context);
}

11
src/Storages/HDFS/WriteBufferFromHDFS.cpp
View File

@ -4,9 +4,9 @@
#include <Storages/HDFS/WriteBufferFromHDFS.h>
#include <Storages/HDFS/HDFSCommon.h>
#include <Common/Throttler.h>
#include <hdfs/hdfs.h>
namespace DB
{
@ -24,15 +24,18 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl
hdfsFile fout;
HDFSBuilderWrapper builder;
HDFSFSPtr fs;
WriteSettings write_settings;
WriteBufferFromHDFSImpl(
const std::string & hdfs_uri_,
const Poco::Util::AbstractConfiguration & config_,
int replication_,
const WriteSettings & write_settings_,
int flags)
: hdfs_uri(hdfs_uri_)
, builder(createHDFSBuilder(hdfs_uri, config_))
, fs(createHDFSFS(builder.get()))
, write_settings(write_settings_)
{
const size_t begin_of_path = hdfs_uri.find('/', hdfs_uri.find("//") + 2);
const String path = hdfs_uri.substr(begin_of_path);
@ -44,7 +47,6 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl
throw Exception("Unable to open HDFS file: " + path + " error: " + std::string(hdfsGetLastError()),
ErrorCodes::CANNOT_OPEN_FILE);
}
}
~WriteBufferFromHDFSImpl()
@ -56,6 +58,8 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl
int write(const char * start, size_t size) const
{
int bytes_written = hdfsWrite(fs.get(), fout, start, size);
if (write_settings.remote_throttler)
write_settings.remote_throttler->add(bytes_written);
if (bytes_written < 0)
throw Exception("Fail to write HDFS file: " + hdfs_uri + " " + std::string(hdfsGetLastError()),
@ -77,10 +81,11 @@ WriteBufferFromHDFS::WriteBufferFromHDFS(
const std::string & hdfs_name_,
const Poco::Util::AbstractConfiguration & config_,
int replication_,
const WriteSettings & write_settings_,
size_t buf_size_,
int flags_)
: BufferWithOwnMemory<WriteBuffer>(buf_size_)
, impl(std::make_unique<WriteBufferFromHDFSImpl>(hdfs_name_, config_, replication_, flags_))
, impl(std::make_unique<WriteBufferFromHDFSImpl>(hdfs_name_, config_, replication_, write_settings_, flags_))
{
}

2
src/Storages/HDFS/WriteBufferFromHDFS.h
View File

@ -4,6 +4,7 @@
#if USE_HDFS
#include <IO/WriteBuffer.h>
#include <IO/WriteSettings.h>
#include <IO/BufferWithOwnMemory.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <fcntl.h>
@ -24,6 +25,7 @@ public:
const String & hdfs_name_,
const Poco::Util::AbstractConfiguration & config_,
int replication_,
const WriteSettings & write_settings_ = {},
size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE,
int flags = O_WRONLY);

5
src/Storages/Hive/StorageHive.cpp
View File

@ -218,7 +218,10 @@ public:
auto get_raw_read_buf = [&]() -> std::unique_ptr<ReadBuffer>
{
auto buf = std::make_unique<ReadBufferFromHDFS>(
hdfs_namenode_url, current_path, getContext()->getGlobalContext()->getConfigRef(), getContext()->getReadSettings());
hdfs_namenode_url,
current_path,
getContext()->getGlobalContext()->getConfigRef(),
getContext()->getReadSettings());
bool thread_pool_read = read_settings.remote_fs_method == RemoteFSReadMethod::threadpool;
if (thread_pool_read)

2
src/Storages/MergeTree/DataPartsExchange.cpp
View File

@ -174,6 +174,8 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write
std::sregex_token_iterator());
if (data_settings->allow_remote_fs_zero_copy_replication &&
/// In memory data part does not have metadata yet.
!isInMemoryPart(part) &&
client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY)
{
auto disk_type = part->data_part_storage->getDiskType();

2
src/Storages/MergeTree/KeyCondition.cpp
View File

@ -2050,7 +2050,7 @@ bool KeyCondition::mayBeTrueInRange(
}
String KeyCondition::RPNElement::toString() const { return toString("column " + std::to_string(key_column), false); }
String KeyCondition::RPNElement::toString(const std::string_view & column_name, bool print_constants) const
String KeyCondition::RPNElement::toString(std::string_view column_name, bool print_constants) const
{
auto print_wrapped_column = [this, &column_name, print_constants](WriteBuffer & buf)
{

2
src/Storages/MergeTree/KeyCondition.h
View File

@ -320,7 +320,7 @@ private:
: function(function_), range(range_), key_column(key_column_) {}
String toString() const;
String toString(const std::string_view & column_name, bool print_constants) const;
String toString(std::string_view column_name, bool print_constants) const;
Function function = FUNCTION_UNKNOWN;

61
src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp
View File

@ -138,18 +138,20 @@ void MergeTreeBackgroundExecutor<Queue>::routine(TaskRuntimeDataPtr item)
}
catch (const Exception & e)
{
NOEXCEPT_SCOPE;
ALLOW_ALLOCATIONS_IN_SCOPE;
if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info.
LOG_INFO(log, fmt::runtime(getCurrentExceptionMessage(false)));
else
tryLogCurrentException(__PRETTY_FUNCTION__);
NOEXCEPT_SCOPE({
ALLOW_ALLOCATIONS_IN_SCOPE;
if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info.
LOG_INFO(log, fmt::runtime(getCurrentExceptionMessage(false)));
else
tryLogCurrentException(__PRETTY_FUNCTION__);
});
}
catch (...)
{
NOEXCEPT_SCOPE;
ALLOW_ALLOCATIONS_IN_SCOPE;
tryLogCurrentException(__PRETTY_FUNCTION__);
NOEXCEPT_SCOPE({
ALLOW_ALLOCATIONS_IN_SCOPE;
tryLogCurrentException(__PRETTY_FUNCTION__);
});
}
if (need_execute_again)
@ -162,9 +164,10 @@ void MergeTreeBackgroundExecutor<Queue>::routine(TaskRuntimeDataPtr item)
/// This is significant to order the destructors.
{
NOEXCEPT_SCOPE;
ALLOW_ALLOCATIONS_IN_SCOPE;
item->task.reset();
NOEXCEPT_SCOPE({
ALLOW_ALLOCATIONS_IN_SCOPE;
item->task.reset();
});
}
item->is_done.set();
item = nullptr;
@ -197,18 +200,20 @@ void MergeTreeBackgroundExecutor<Queue>::routine(TaskRuntimeDataPtr item)
}
catch (const Exception & e)
{
NOEXCEPT_SCOPE;
ALLOW_ALLOCATIONS_IN_SCOPE;
if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info.
LOG_INFO(log, fmt::runtime(getCurrentExceptionMessage(false)));
else
tryLogCurrentException(__PRETTY_FUNCTION__);
NOEXCEPT_SCOPE({
ALLOW_ALLOCATIONS_IN_SCOPE;
if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info.
LOG_INFO(log, fmt::runtime(getCurrentExceptionMessage(false)));
else
tryLogCurrentException(__PRETTY_FUNCTION__);
});
}
catch (...)
{
NOEXCEPT_SCOPE;
ALLOW_ALLOCATIONS_IN_SCOPE;
tryLogCurrentException(__PRETTY_FUNCTION__);
NOEXCEPT_SCOPE({
ALLOW_ALLOCATIONS_IN_SCOPE;
tryLogCurrentException(__PRETTY_FUNCTION__);
});
}
@ -218,9 +223,10 @@ void MergeTreeBackgroundExecutor<Queue>::routine(TaskRuntimeDataPtr item)
/// The thread that shutdowns storage will scan queues in order to find some tasks to wait for, but will find nothing.
/// So, the destructor of a task and the destructor of a storage will be executed concurrently.
{
NOEXCEPT_SCOPE;
ALLOW_ALLOCATIONS_IN_SCOPE;
item->task.reset();
NOEXCEPT_SCOPE({
ALLOW_ALLOCATIONS_IN_SCOPE;
item->task.reset();
});
}
item->is_done.set();
@ -256,9 +262,10 @@ void MergeTreeBackgroundExecutor<Queue>::threadFunction()
}
catch (...)
{
NOEXCEPT_SCOPE;
ALLOW_ALLOCATIONS_IN_SCOPE;
tryLogCurrentException(__PRETTY_FUNCTION__);
NOEXCEPT_SCOPE({
ALLOW_ALLOCATIONS_IN_SCOPE;
tryLogCurrentException(__PRETTY_FUNCTION__);
});
}
}
}

118
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -4942,78 +4942,78 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData:
}
}
NOEXCEPT_SCOPE;
NOEXCEPT_SCOPE({
auto current_time = time(nullptr);
auto current_time = time(nullptr);
size_t add_bytes = 0;
size_t add_rows = 0;
size_t add_parts = 0;
size_t add_bytes = 0;
size_t add_rows = 0;
size_t add_parts = 0;
size_t reduce_bytes = 0;
size_t reduce_rows = 0;
size_t reduce_parts = 0;
size_t reduce_bytes = 0;
size_t reduce_rows = 0;
size_t reduce_parts = 0;
for (const DataPartPtr & part : precommitted_parts)
{
auto part_in_memory = asInMemoryPart(part);
if (part_in_memory && settings->in_memory_parts_enable_wal)
for (const DataPartPtr & part : precommitted_parts)
{
if (!wal)
wal = data.getWriteAheadLog();
wal->addPart(part_in_memory);
}
DataPartPtr covering_part;
DataPartsVector covered_parts = data.getActivePartsToReplace(part->info, part->name, covering_part, *owing_parts_lock);
if (covering_part)
{
LOG_WARNING(data.log, "Tried to commit obsolete part {} covered by {}", part->name, covering_part->getNameWithState());
part->remove_time.store(0, std::memory_order_relaxed); /// The part will be removed without waiting for old_parts_lifetime seconds.
data.modifyPartState(part, DataPartState::Outdated);
}
else
{
if (!txn)
MergeTreeTransaction::addNewPartAndRemoveCovered(data.shared_from_this(), part, covered_parts, NO_TRANSACTION_RAW);
total_covered_parts.insert(total_covered_parts.end(), covered_parts.begin(), covered_parts.end());
for (const auto & covered_part : covered_parts)
auto part_in_memory = asInMemoryPart(part);
if (part_in_memory && settings->in_memory_parts_enable_wal)
{
covered_part->remove_time.store(current_time, std::memory_order_relaxed);
if (!wal)
wal = data.getWriteAheadLog();
reduce_bytes += covered_part->getBytesOnDisk();
reduce_rows += covered_part->rows_count;
data.modifyPartState(covered_part, DataPartState::Outdated);
data.removePartContributionToColumnAndSecondaryIndexSizes(covered_part);
wal->addPart(part_in_memory);
}
reduce_parts += covered_parts.size();
DataPartPtr covering_part;
DataPartsVector covered_parts = data.getActivePartsToReplace(part->info, part->name, covering_part, *owing_parts_lock);
if (covering_part)
{
LOG_WARNING(data.log, "Tried to commit obsolete part {} covered by {}", part->name, covering_part->getNameWithState());
add_bytes += part->getBytesOnDisk();
add_rows += part->rows_count;
++add_parts;
part->remove_time.store(0, std::memory_order_relaxed); /// The part will be removed without waiting for old_parts_lifetime seconds.
data.modifyPartState(part, DataPartState::Outdated);
}
else
{
if (!txn)
MergeTreeTransaction::addNewPartAndRemoveCovered(data.shared_from_this(), part, covered_parts, NO_TRANSACTION_RAW);
data.modifyPartState(part, DataPartState::Active);
data.addPartContributionToColumnAndSecondaryIndexSizes(part);
total_covered_parts.insert(total_covered_parts.end(), covered_parts.begin(), covered_parts.end());
for (const auto & covered_part : covered_parts)
{
covered_part->remove_time.store(current_time, std::memory_order_relaxed);
reduce_bytes += covered_part->getBytesOnDisk();
reduce_rows += covered_part->rows_count;
data.modifyPartState(covered_part, DataPartState::Outdated);
data.removePartContributionToColumnAndSecondaryIndexSizes(covered_part);
}
reduce_parts += covered_parts.size();
add_bytes += part->getBytesOnDisk();
add_rows += part->rows_count;
++add_parts;
data.modifyPartState(part, DataPartState::Active);
data.addPartContributionToColumnAndSecondaryIndexSizes(part);
}
}
}
if (reduce_parts == 0)
{
for (const auto & part : precommitted_parts)
data.updateObjectColumns(part, parts_lock);
}
else
data.resetObjectColumnsFromActiveParts(parts_lock);
if (reduce_parts == 0)
{
for (const auto & part : precommitted_parts)
data.updateObjectColumns(part, parts_lock);
}
else
data.resetObjectColumnsFromActiveParts(parts_lock);
ssize_t diff_bytes = add_bytes - reduce_bytes;
ssize_t diff_rows = add_rows - reduce_rows;
ssize_t diff_parts = add_parts - reduce_parts;
data.increaseDataVolume(diff_bytes, diff_rows, diff_parts);
ssize_t diff_bytes = add_bytes - reduce_bytes;
ssize_t diff_rows = add_rows - reduce_rows;
ssize_t diff_parts = add_parts - reduce_parts;
data.increaseDataVolume(diff_bytes, diff_rows, diff_parts);
});
}
clear();

5
src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp
View File

@ -1102,9 +1102,8 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry
if (future_part.isDisjoint(result_part))
continue;
/// Parts are not disjoint, so new_part_name either contains or covers future_part.
if (!(future_part.contains(result_part) || result_part.contains(future_part)))
throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected non-disjoint parts: {} and {}", future_part_elem.first, new_part_name);
/// Parts are not disjoint. They can be even intersecting and it's not a problem,
/// because we may have two queue entries producing intersecting parts if there's DROP_RANGE between them (so virtual_parts are ok).
/// We cannot execute `entry` (or upgrade its actual_part_name to `new_part_name`)
/// while any covered or covering parts are processed.

3
src/Storages/StorageS3.cpp
View File

@ -587,7 +587,8 @@ public:
s3_configuration_.rw_settings,
std::nullopt,
DBMS_DEFAULT_BUFFER_SIZE,
threadPoolCallbackRunner(IOThreadPool::get())),
threadPoolCallbackRunner(IOThreadPool::get()),
context->getWriteSettings()),
compression_method,
3);
writer

2
src/Storages/System/StorageSystemPrivileges.cpp
View File

@ -85,7 +85,7 @@ void StorageSystemPrivileges::fillData(MutableColumns & res_columns, ContextPtr,
auto & column_parent_group = assert_cast<ColumnInt16 &>(assert_cast<ColumnNullable &>(*res_columns[column_index]).getNestedColumn()).getData();
auto & column_parent_group_null_map = assert_cast<ColumnNullable &>(*res_columns[column_index++]).getNullMapData();
auto add_row = [&](AccessType access_type, const std::string_view & aliases, Level max_level, AccessType parent_group)
auto add_row = [&](AccessType access_type, std::string_view aliases, Level max_level, AccessType parent_group)
{
column_access_type.push_back(static_cast<Int16>(access_type));

1
tests/integration/test_s3_zero_copy_replication/test.py
View File

@ -150,6 +150,7 @@ def test_s3_zero_copy_replication(cluster, policy):
node2.query("DROP TABLE IF EXISTS s3_test NO DELAY")
@pytest.mark.skip(reason="Test is flaky (and never was stable)")
def test_s3_zero_copy_on_hybrid_storage(cluster):
node1 = cluster.instances["node1"]
node2 = cluster.instances["node2"]

Some files were not shown because too many files have changed in this diff Show More