mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 01:25:21 +00:00
Merge master
This commit is contained in:
kssenii
commit
8ad26b3111
16
.github/workflows/master.yml
vendored
16
.github/workflows/master.yml
vendored
@ -2902,7 +2902,7 @@ jobs:
|
||||
#############################################################################################
|
||||
#################################### PERFORMANCE TESTS ######################################
|
||||
#############################################################################################
|
||||
PerformanceComparison0:
|
||||
PerformanceComparisonX86-0:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -2940,7 +2940,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison1:
|
||||
PerformanceComparisonX86-1:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -2978,7 +2978,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison2:
|
||||
PerformanceComparisonX86-2:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3016,7 +3016,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison3:
|
||||
PerformanceComparisonX86-3:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3096,10 +3096,10 @@ jobs:
|
||||
- IntegrationTestsTsan1
|
||||
- IntegrationTestsTsan2
|
||||
- IntegrationTestsTsan3
|
||||
- PerformanceComparison0
|
||||
- PerformanceComparison1
|
||||
- PerformanceComparison2
|
||||
- PerformanceComparison3
|
||||
- PerformanceComparisonX86-0
|
||||
- PerformanceComparisonX86-1
|
||||
- PerformanceComparisonX86-2
|
||||
- PerformanceComparisonX86-3
|
||||
- CompatibilityCheck
|
||||
- ASTFuzzerTestDebug
|
||||
- ASTFuzzerTestAsan
|
||||
|
||||
32
.github/workflows/pull_request.yml
vendored
32
.github/workflows/pull_request.yml
vendored
@ -3118,7 +3118,7 @@ jobs:
|
||||
#############################################################################################
|
||||
#################################### PERFORMANCE TESTS ######################################
|
||||
#############################################################################################
|
||||
PerformanceComparison0:
|
||||
PerformanceComparisonX86-0:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3156,7 +3156,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison1:
|
||||
PerformanceComparisonX86-1:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3194,7 +3194,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison2:
|
||||
PerformanceComparisonX86-2:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3232,7 +3232,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison3:
|
||||
PerformanceComparisonX86-3:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3270,7 +3270,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparisonAarch0:
|
||||
PerformanceComparisonAarch-0:
|
||||
needs: [BuilderDebAarch64]
|
||||
runs-on: [self-hosted, func-tester-aarch64]
|
||||
steps:
|
||||
@ -3308,7 +3308,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparisonAarch1:
|
||||
PerformanceComparisonAarch-1:
|
||||
needs: [BuilderDebAarch64]
|
||||
runs-on: [self-hosted, func-tester-aarch64]
|
||||
steps:
|
||||
@ -3346,7 +3346,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparisonAarch2:
|
||||
PerformanceComparisonAarch-2:
|
||||
needs: [BuilderDebAarch64]
|
||||
runs-on: [self-hosted, func-tester-aarch64]
|
||||
steps:
|
||||
@ -3384,7 +3384,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparisonAarch3:
|
||||
PerformanceComparisonAarch-3:
|
||||
needs: [BuilderDebAarch64]
|
||||
runs-on: [self-hosted, func-tester-aarch64]
|
||||
steps:
|
||||
@ -3481,14 +3481,14 @@ jobs:
|
||||
- IntegrationTestsTsan1
|
||||
- IntegrationTestsTsan2
|
||||
- IntegrationTestsTsan3
|
||||
- PerformanceComparison0
|
||||
- PerformanceComparison1
|
||||
- PerformanceComparison2
|
||||
- PerformanceComparison3
|
||||
- PerformanceComparisonAarch0
|
||||
- PerformanceComparisonAarch1
|
||||
- PerformanceComparisonAarch2
|
||||
- PerformanceComparisonAarch3
|
||||
- PerformanceComparisonX86-0
|
||||
- PerformanceComparisonX86-1
|
||||
- PerformanceComparisonX86-2
|
||||
- PerformanceComparisonX86-3
|
||||
- PerformanceComparisonAarch-0
|
||||
- PerformanceComparisonAarch-1
|
||||
- PerformanceComparisonAarch-2
|
||||
- PerformanceComparisonAarch-3
|
||||
- UnitTestsAsan
|
||||
- UnitTestsTsan
|
||||
- UnitTestsMsan
|
||||
|
||||
@ -335,6 +335,22 @@ if (COMPILER_GCC OR COMPILER_CLANG)
|
||||
set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
|
||||
endif ()
|
||||
|
||||
if (ARCH_AMD64)
|
||||
# align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
|
||||
# which makes benchmark results more stable.
|
||||
set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
|
||||
if (COMPILER_GCC)
|
||||
# gcc is in assembler, need to add "-Wa," prefix
|
||||
set(BRANCHES_WITHIN_32B_BOUNDARIES "-Wa,${BRANCHES_WITHIN_32B_BOUNDARIES}")
|
||||
endif()
|
||||
|
||||
include(CheckCXXCompilerFlag)
|
||||
check_cxx_compiler_flag("${BRANCHES_WITHIN_32B_BOUNDARIES}" HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
|
||||
if (HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
|
||||
set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (COMPILER_GCC)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines")
|
||||
endif ()
|
||||
|
||||
@ -12,7 +12,14 @@
|
||||
#define JSON_MAX_DEPTH 100
|
||||
|
||||
|
||||
#ifdef __clang__
|
||||
# pragma clang diagnostic push
|
||||
# pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec"
|
||||
#endif
|
||||
POCO_IMPLEMENT_EXCEPTION(JSONException, Poco::Exception, "JSONException") // NOLINT(cert-err60-cpp, modernize-use-noexcept, hicpp-use-noexcept)
|
||||
#ifdef __clang__
|
||||
# pragma clang diagnostic pop
|
||||
#endif
|
||||
|
||||
|
||||
/// Прочитать беззнаковое целое в простом формате из не-0-terminated строки.
|
||||
|
||||
@ -38,8 +38,14 @@
|
||||
*/
|
||||
|
||||
|
||||
#ifdef __clang__
|
||||
# pragma clang diagnostic push
|
||||
# pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec"
|
||||
#endif
|
||||
POCO_DECLARE_EXCEPTION(Foundation_API, JSONException, Poco::Exception)
|
||||
|
||||
#ifdef __clang__
|
||||
# pragma clang diagnostic pop
|
||||
#endif
|
||||
|
||||
class JSON
|
||||
{
|
||||
|
||||
@ -20,16 +20,11 @@ if (COMPILER_CLANG)
|
||||
# We want to get everything out of the compiler for code quality.
|
||||
add_warning(everything)
|
||||
add_warning(pedantic)
|
||||
no_warning(vla-extension)
|
||||
no_warning(zero-length-array)
|
||||
no_warning(c11-extensions)
|
||||
no_warning(unused-command-line-argument)
|
||||
no_warning(c++98-compat-pedantic)
|
||||
no_warning(c++98-compat)
|
||||
no_warning(c99-extensions)
|
||||
no_warning(conversion)
|
||||
no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
|
||||
no_warning(deprecated-dynamic-exception-spec)
|
||||
no_warning(disabled-macro-expansion)
|
||||
no_warning(documentation-unknown-command)
|
||||
no_warning(double-promotion)
|
||||
@ -38,12 +33,7 @@ if (COMPILER_CLANG)
|
||||
no_warning(global-constructors)
|
||||
no_warning(missing-prototypes)
|
||||
no_warning(missing-variable-declarations)
|
||||
no_warning(nested-anon-types)
|
||||
no_warning(packed)
|
||||
no_warning(padded)
|
||||
no_warning(return-std-move-in-c++11) # clang 7+
|
||||
no_warning(shift-sign-overflow)
|
||||
no_warning(sign-conversion)
|
||||
no_warning(switch-enum)
|
||||
no_warning(undefined-func-template)
|
||||
no_warning(unused-template)
|
||||
|
||||
2
contrib/thrift
vendored
2
contrib/thrift
vendored
@ -1 +1 @@
|
||||
Subproject commit 010ccf0a0c7023fea0f6bf4e4078ebdff7e61982
|
||||
Subproject commit 2a93df80f27739ccabb5b885cb12a8dc7595ecdf
|
||||
@ -15,7 +15,6 @@ set(thriftcpp_SOURCES
|
||||
"${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/concurrency/ThreadManager.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/concurrency/TimerManager.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/concurrency/Util.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/processor/PeekProcessor.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/protocol/TBase64Utils.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/protocol/TDebugProtocol.cpp"
|
||||
@ -33,6 +32,8 @@ set(thriftcpp_SOURCES
|
||||
"${LIBRARY_DIR}/src/thrift/transport/TServerSocket.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/transport/TTransportUtils.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/transport/TBufferTransports.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/transport/SocketCommon.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/transport/TWebSocketServer.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/server/TConnectedClient.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/server/TServerFramework.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/server/TSimpleServer.cpp"
|
||||
@ -92,4 +93,4 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}")
|
||||
add_library(_thrift ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
|
||||
add_library(ch_contrib::thrift ALIAS _thrift)
|
||||
target_include_directories(_thrift SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src" ${CMAKE_CURRENT_BINARY_DIR})
|
||||
target_link_libraries (_thrift PUBLIC boost::headers_only)
|
||||
target_link_libraries (_thrift PUBLIC OpenSSL::SSL boost::headers_only)
|
||||
|
||||
@ -29,17 +29,21 @@ env
|
||||
|
||||
if [ -n "$MAKE_DEB" ]; then
|
||||
rm -rf /build/packages/root
|
||||
if [ -z "$SANITIZER" ]; then
|
||||
# We need to check if clickhouse-diagnostics is fine and build it
|
||||
(
|
||||
cd /build/programs/diagnostics
|
||||
make test-no-docker
|
||||
GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build
|
||||
mv clickhouse-diagnostics ..
|
||||
)
|
||||
else
|
||||
echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics
|
||||
chmod +x /build/programs/clickhouse-diagnostics
|
||||
# NOTE: this is for backward compatibility with previous releases,
|
||||
# that does not diagnostics tool (only script).
|
||||
if [ -d /build/programs/diagnostics ]; then
|
||||
if [ -z "$SANITIZER" ]; then
|
||||
# We need to check if clickhouse-diagnostics is fine and build it
|
||||
(
|
||||
cd /build/programs/diagnostics
|
||||
make test-no-docker
|
||||
GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build
|
||||
mv clickhouse-diagnostics ..
|
||||
)
|
||||
else
|
||||
echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics
|
||||
chmod +x /build/programs/clickhouse-diagnostics
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@ -71,6 +71,7 @@ RUN apt-get update \
|
||||
software-properties-common \
|
||||
tzdata \
|
||||
unixodbc \
|
||||
file \
|
||||
--yes --no-install-recommends
|
||||
|
||||
RUN pip3 install numpy scipy pandas Jinja2
|
||||
|
||||
@ -34,6 +34,7 @@ RUN apt-get update -y \
|
||||
unixodbc \
|
||||
wget \
|
||||
zstd \
|
||||
file \
|
||||
&& apt-get clean
|
||||
|
||||
|
||||
|
||||
@ -324,7 +324,11 @@ else
|
||||
clickhouse-client --query="SELECT 'Server version: ', version()"
|
||||
|
||||
# Install new package before running stress test because we should use new clickhouse-client and new clickhouse-test
|
||||
# But we should leave old binary in /usr/bin/ for gdb (so it will print sane stacktarces)
|
||||
mv /usr/bin/clickhouse previous_release_package_folder/
|
||||
install_packages package_folder
|
||||
mv /usr/bin/clickhouse package_folder/
|
||||
mv previous_release_package_folder/clickhouse /usr/bin/
|
||||
|
||||
mkdir tmp_stress_output
|
||||
|
||||
@ -339,6 +343,7 @@ else
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
|
||||
|
||||
# Start new server
|
||||
mv package_folder/clickhouse /usr/bin/
|
||||
configure
|
||||
start 500
|
||||
clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
|
||||
@ -356,6 +361,9 @@ else
|
||||
# Error messages (we should ignore some errors)
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64")
|
||||
# FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
|
||||
# Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
|
||||
echo "Check for Error messages in server log:"
|
||||
zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
|
||||
-e "Code: 236. DB::Exception: Cancelled mutating parts" \
|
||||
@ -380,6 +388,9 @@ else
|
||||
-e "is lost forever." \
|
||||
-e "Unknown index: idx." \
|
||||
-e "Cannot parse string 'Hello' as UInt64" \
|
||||
-e "} <Error> TCPHandler: Code:" \
|
||||
-e "} <Error> executeQuery: Code:" \
|
||||
-e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
|
||||
/var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
|
||||
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
@ -19,6 +19,10 @@ The `system.part_log` table contains the following columns:
|
||||
- `REGULAR_MERGE` — Some regular merge.
|
||||
- `TTL_DELETE_MERGE` — Cleaning up expired data.
|
||||
- `TTL_RECOMPRESS_MERGE` — Recompressing data part with the.
|
||||
- `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Merge algorithm for the event with type `MERGE_PARTS`. Can have one of the following values:
|
||||
- `UNDECIDED`
|
||||
- `HORIZONTAL`
|
||||
- `VERTICAL`
|
||||
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
|
||||
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
|
||||
- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision.
|
||||
@ -52,6 +56,7 @@ Row 1:
|
||||
query_id: 983ad9c7-28d5-4ae1-844e-603116b7de31
|
||||
event_type: NewPart
|
||||
merge_reason: NotAMerge
|
||||
merge_algorithm: Undecided
|
||||
event_date: 2021-02-02
|
||||
event_time: 2021-02-02 11:14:28
|
||||
event_time_microseconds: 2021-02-02 11:14:28.861919
|
||||
|
||||
@ -1715,3 +1715,7 @@ Return value type is always [Float64](../../sql-reference/data-types/float.md).
|
||||
│ 6 │ Float64 │
|
||||
└─────┴──────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Distance functions
|
||||
|
||||
All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).
|
||||
|
||||
502
docs/en/sql-reference/functions/distance-functions.md
Normal file
502
docs/en/sql-reference/functions/distance-functions.md
Normal file
@ -0,0 +1,502 @@
|
||||
# Distance functions
|
||||
|
||||
## L1Norm
|
||||
|
||||
Calculates the sum of absolute values of a vector.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Norm(vector)
|
||||
```
|
||||
|
||||
Alias: `normL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance.
|
||||
|
||||
Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Norm((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Norm((1, 2))─┐
|
||||
│ 3 │
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
## L2Norm
|
||||
|
||||
Calculates the square root of the sum of the squares of the vector values.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Norm(vector)
|
||||
```
|
||||
|
||||
Alias: `normL2`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance).
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Norm((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌───L2Norm((1, 2))─┐
|
||||
│ 2.23606797749979 │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
## LinfNorm
|
||||
|
||||
Calculates the maximum of absolute values of a vector.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfNorm(vector)
|
||||
```
|
||||
|
||||
Alias: `normLinf`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Linf-norm or the maximum absolute value.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfNorm((1, -2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfNorm((1, -2))─┐
|
||||
│ 2 │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
## LpNorm
|
||||
|
||||
Calculates the root of `p`-th power of the sum of the absolute values of a vector in the power of `p`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpNorm(vector, p)
|
||||
```
|
||||
|
||||
Alias: `normLp`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `p` — The power. Possible values: real number in `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpNorm((1, -2), 2);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpNorm((1, -2), 2)─┐
|
||||
│ 2.23606797749979 │
|
||||
└────────────────────┘
|
||||
```
|
||||
|
||||
## L1Distance
|
||||
|
||||
Calculates the distance between two points (the values of the vectors are the coordinates) in `L1` space (1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Distance(vector1, vector2)
|
||||
```
|
||||
|
||||
Alias: `distanceL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- 1-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Distance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Distance((1, 2), (2, 3))─┐
|
||||
│ 2 │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## L2Distance
|
||||
|
||||
Calculates the distance between two points (the values of the vectors are the coordinates) in Euclidean space ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Distance(vector1, vector2)
|
||||
```
|
||||
|
||||
Alias: `distanceL2`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- 2-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Distance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L2Distance((1, 2), (2, 3))─┐
|
||||
│ 1.4142135623730951 │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## LinfDistance
|
||||
|
||||
Calculates the distance between two points (the values of the vectors are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfDistance(vector1, vector2)
|
||||
```
|
||||
|
||||
Alias: `distanceLinf`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `vector1` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Infinity-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfDistance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfDistance((1, 2), (2, 3))─┐
|
||||
│ 1 │
|
||||
└──────────────────────────────┘
|
||||
```
|
||||
|
||||
## LpDistance
|
||||
|
||||
Calculates the distance between two points (the values of the vectors are the coordinates) in `Lp` space ([p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpDistance(vector1, vector2, p)
|
||||
```
|
||||
|
||||
Alias: `distanceLp`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `p` — The power. Possible values: real number from `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- p-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpDistance((1, 2), (2, 3), 3);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpDistance((1, 2), (2, 3), 3)─┐
|
||||
│ 1.2599210498948732 │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
## L1Normalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L1` space ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Normalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Normalize((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Normalize((1, 2))─────────────────────┐
|
||||
│ (0.3333333333333333,0.6666666666666666) │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## L2Normalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in Euclidean space (using [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Normalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Normalize((3, 4));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L2Normalize((3, 4))─┐
|
||||
│ (0.6,0.8) │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## LinfNormalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L_{inf}` space (using [maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfNormalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeLinf `.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfNormalize((3, 4));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfNormalize((3, 4))─┐
|
||||
│ (0.75,1) │
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
## LpNormalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `Lp` space (using [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpNormalize(tuple, p)
|
||||
```
|
||||
|
||||
Alias: `normalizeLp `.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpNormalize((3, 4),5);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpNormalize((3, 4), 5)──────────────────┐
|
||||
│ (0.7187302630182624,0.9583070173576831) │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## cosineDistance
|
||||
|
||||
Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
cosineDistance(tuple1, tuple2)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Cosine of the angle between two vectors substracted from one.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT cosineDistance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─cosineDistance((1, 2), (2, 3))─┐
|
||||
│ 0.007722123286332261 │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
@ -93,3 +93,35 @@ Predefined characters: `\0`, `\\`, `|`, `(`, `)`, `^`, `$`, `.`, `[`, `]`, `?`,
|
||||
This implementation slightly differs from re2::RE2::QuoteMeta. It escapes zero byte as `\0` instead of `\x00` and it escapes only required characters.
|
||||
For more information, see the link: [RE2](https://github.com/google/re2/blob/master/re2/re2.cc#L473)
|
||||
|
||||
|
||||
## translate(s, from, to)
|
||||
|
||||
The function replaces characters in the string ‘s’ in accordance with one-to-one character mapping defined by ‘from’ and ‘to’ strings. ‘from’ and ‘to’ must be ASCII strings of the same size. Non-ASCII characters in the original string are not modified.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT translate('Hello, World!', 'delor', 'DELOR') AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ HELLO, WORLD! │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
## translateUTF8(string, from, to)
|
||||
|
||||
Similar to previous function, but works with UTF-8 arguments. ‘from’ and ‘to’ must be valid UTF-8 strings of the same size.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT translateUTF8('Hélló, Wórld¡', 'óé¡', 'oe!') AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ Hello, World! │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
@ -374,7 +374,7 @@ The same as `multiMatchAny`, but returns any index that matches the haystack.
|
||||
|
||||
## multiMatchAllIndices(haystack, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
|
||||
|
||||
The same as `multiMatchAny`, but returns the array of all indicies that match the haystack in any order.
|
||||
The same as `multiMatchAny`, but returns the array of all indices that match the haystack in any order.
|
||||
|
||||
## multiFuzzyMatchAny(haystack, distance, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
|
||||
|
||||
|
||||
@ -559,502 +559,7 @@ Result:
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## L1Norm
|
||||
|
||||
Calculates the sum of absolute values of a tuple.
|
||||
## Distance functions
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Norm(tuple)
|
||||
```
|
||||
|
||||
Alias: `normL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance.
|
||||
|
||||
Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Norm((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Norm((1, 2))─┐
|
||||
│ 3 │
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
## L2Norm
|
||||
|
||||
Calculates the square root of the sum of the squares of the tuple values.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Norm(tuple)
|
||||
```
|
||||
|
||||
Alias: `normL2`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance).
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Norm((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌───L2Norm((1, 2))─┐
|
||||
│ 2.23606797749979 │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
## LinfNorm
|
||||
|
||||
Calculates the maximum of absolute values of a tuple.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfNorm(tuple)
|
||||
```
|
||||
|
||||
Alias: `normLinf`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Linf-norm or the maximum absolute value.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfNorm((1, -2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfNorm((1, -2))─┐
|
||||
│ 2 │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
## LpNorm
|
||||
|
||||
Calculates the root of `p`-th power of the sum of the absolute values of a tuple in the power of `p`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpNorm(tuple, p)
|
||||
```
|
||||
|
||||
Alias: `normLp`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `p` — The power. Possible values: real number in `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpNorm((1, -2), 2);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpNorm((1, -2), 2)─┐
|
||||
│ 2.23606797749979 │
|
||||
└────────────────────┘
|
||||
```
|
||||
|
||||
## L1Distance
|
||||
|
||||
Calculates the distance between two points (the values of the tuples are the coordinates) in `L1` space (1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Distance(tuple1, tuple2)
|
||||
```
|
||||
|
||||
Alias: `distanceL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- 1-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Distance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Distance((1, 2), (2, 3))─┐
|
||||
│ 2 │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## L2Distance
|
||||
|
||||
Calculates the distance between two points (the values of the tuples are the coordinates) in Euclidean space ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Distance(tuple1, tuple2)
|
||||
```
|
||||
|
||||
Alias: `distanceL2`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- 2-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Distance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L2Distance((1, 2), (2, 3))─┐
|
||||
│ 1.4142135623730951 │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## LinfDistance
|
||||
|
||||
Calculates the distance between two points (the values of the tuples are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfDistance(tuple1, tuple2)
|
||||
```
|
||||
|
||||
Alias: `distanceLinf`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Infinity-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfDistance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfDistance((1, 2), (2, 3))─┐
|
||||
│ 1 │
|
||||
└──────────────────────────────┘
|
||||
```
|
||||
|
||||
## LpDistance
|
||||
|
||||
Calculates the distance between two points (the values of the tuples are the coordinates) in `Lp` space ([p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpDistance(tuple1, tuple2, p)
|
||||
```
|
||||
|
||||
Alias: `distanceLp`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `p` — The power. Possible values: real number from `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- p-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpDistance((1, 2), (2, 3), 3);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpDistance((1, 2), (2, 3), 3)─┐
|
||||
│ 1.2599210498948732 │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
## L1Normalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L1` space ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Normalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Normalize((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Normalize((1, 2))─────────────────────┐
|
||||
│ (0.3333333333333333,0.6666666666666666) │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## L2Normalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in Euclidean space (using [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Normalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Normalize((3, 4));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L2Normalize((3, 4))─┐
|
||||
│ (0.6,0.8) │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## LinfNormalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L_{inf}` space (using [maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfNormalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeLinf `.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfNormalize((3, 4));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfNormalize((3, 4))─┐
|
||||
│ (0.75,1) │
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
## LpNormalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `Lp` space (using [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpNormalize(tuple, p)
|
||||
```
|
||||
|
||||
Alias: `normalizeLp `.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpNormalize((3, 4),5);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpNormalize((3, 4), 5)──────────────────┐
|
||||
│ (0.7187302630182624,0.9583070173576831) │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## cosineDistance
|
||||
|
||||
Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
cosineDistance(tuple1, tuple2)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Cosine of the angle between two vectors substracted from one.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT cosineDistance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─cosineDistance((1, 2), (2, 3))─┐
|
||||
│ 0.007722123286332261 │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).
|
||||
|
||||
@ -5,7 +5,7 @@ sidebar_label: null function
|
||||
|
||||
# null
|
||||
|
||||
Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately droped right after the query execution. The function is used for the convenience of test writing and demonstrations.
|
||||
Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately dropped right after the query execution. The function is used for the convenience of test writing and demonstrations.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -40,4 +40,3 @@ See also:
|
||||
|
||||
- [Null table engine](../../engines/table-engines/special/null.md)
|
||||
|
||||
[Original article](https://clickhouse.com/docs/en/sql-reference/table-functions/null/) <!--hide-->
|
||||
|
||||
@ -5,11 +5,11 @@ sidebar_label: remote
|
||||
|
||||
# remote, remoteSecure
|
||||
|
||||
Allows to access remote servers without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. `remoteSecure` - same as `remote` but with a secured connection.
|
||||
Allows accessing remote servers, including migration of data, without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. `remoteSecure` - same as `remote` but with a secured connection.
|
||||
|
||||
Both functions can be used in `SELECT` and `INSERT` queries.
|
||||
|
||||
**Syntax**
|
||||
## Syntax
|
||||
|
||||
``` sql
|
||||
remote('addresses_expr', db, table[, 'user'[, 'password'], sharding_key])
|
||||
@ -18,7 +18,7 @@ remoteSecure('addresses_expr', db, table[, 'user'[, 'password'], sharding_key])
|
||||
remoteSecure('addresses_expr', db.table[, 'user'[, 'password'], sharding_key])
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
## Parameters
|
||||
|
||||
- `addresses_expr` — An expression that generates addresses of remote servers. This may be just one server address. The server address is `host:port`, or just `host`.
|
||||
|
||||
@ -36,28 +36,31 @@ remoteSecure('addresses_expr', db.table[, 'user'[, 'password'], sharding_key])
|
||||
- `password` — User password. If the password is not specified, an empty password is used. Type: [String](../../sql-reference/data-types/string.md).
|
||||
- `sharding_key` — Sharding key to support distributing data across nodes. For example: `insert into remote('127.0.0.1:9000,127.0.0.2', db, table, 'default', rand())`. Type: [UInt32](../../sql-reference/data-types/int-uint.md).
|
||||
|
||||
**Returned value**
|
||||
## Returned value
|
||||
|
||||
The dataset from remote servers.
|
||||
|
||||
**Usage**
|
||||
## Usage
|
||||
|
||||
Using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function.
|
||||
Unless you are migrating data from one system to another, using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function.
|
||||
|
||||
The `remote` table function can be useful in the following cases:
|
||||
|
||||
- Accessing a specific server for data comparison, debugging, and testing.
|
||||
- Queries between various ClickHouse clusters for research purposes.
|
||||
- Infrequent distributed requests that are made manually.
|
||||
- Distributed requests where the set of servers is re-defined each time.
|
||||
- Migrating data from one system to another
|
||||
- Accessing a specific server for data comparison, debugging, and testing.
|
||||
- Queries between various ClickHouse clusters for research purposes.
|
||||
- Infrequent distributed requests that are made manually.
|
||||
- Distributed requests where the set of servers is re-defined each time.
|
||||
|
||||
**Adresses**
|
||||
### Adresses
|
||||
|
||||
``` text
|
||||
example01-01-1
|
||||
example01-01-1:9440
|
||||
example01-01-1:9000
|
||||
localhost
|
||||
127.0.0.1
|
||||
[::]:9440
|
||||
[::]:9000
|
||||
[2a02:6b8:0:1111::11]:9000
|
||||
```
|
||||
@ -68,15 +71,15 @@ Multiple addresses can be comma-separated. In this case, ClickHouse will use dis
|
||||
example01-01-1,example01-02-1
|
||||
```
|
||||
|
||||
**Examples**
|
||||
## Examples
|
||||
|
||||
Selecting data from a remote server:
|
||||
### Selecting data from a remote server:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM remote('127.0.0.1', db.remote_engine_table) LIMIT 3;
|
||||
```
|
||||
|
||||
Inserting data from a remote server into a table:
|
||||
### Inserting data from a remote server into a table:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE remote_table (name String, value UInt32) ENGINE=Memory;
|
||||
@ -84,7 +87,65 @@ INSERT INTO FUNCTION remote('127.0.0.1', currentDatabase(), 'remote_table') VALU
|
||||
SELECT * FROM remote_table;
|
||||
```
|
||||
|
||||
## Globs in Addresses {globs-in-addresses}
|
||||
### Migration of tables from one system to another:
|
||||
This example uses one table from a sample dataset. The database is `imdb`, and the table is `actors`.
|
||||
|
||||
#### On the source ClickHouse system (the system that currently hosts the data)
|
||||
- Verify the source database and table name (`imdb.actors`)
|
||||
```sql
|
||||
show databases
|
||||
```
|
||||
|
||||
```sql
|
||||
show tables in imdb
|
||||
```
|
||||
|
||||
- Get the CREATE TABLE statement from the source:
|
||||
```
|
||||
select create_table_query
|
||||
from system.tables
|
||||
where database = 'imdb' and table = 'actors'
|
||||
```
|
||||
|
||||
Response
|
||||
```sql
|
||||
CREATE TABLE imdb.actors (`id` UInt32,
|
||||
`first_name` String,
|
||||
`last_name` String,
|
||||
`gender` FixedString(1))
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
|
||||
ORDER BY (id, first_name, last_name, gender)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
#### On the destination ClickHouse system:
|
||||
|
||||
- Create the destination database:
|
||||
```sql
|
||||
CREATE DATABASE imdb
|
||||
```
|
||||
|
||||
- Using the CREATE TABLE statement from the source, create the destination:
|
||||
```sql
|
||||
CREATE TABLE imdb.actors (`id` UInt32,
|
||||
`first_name` String,
|
||||
`last_name` String,
|
||||
`gender` FixedString(1))
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
|
||||
ORDER BY (id, first_name, last_name, gender)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
#### Back on the source deployment:
|
||||
|
||||
Insert into the new database and table created on the remote system. You will need the host, port, username, password, destination database, and destination table.
|
||||
```sql
|
||||
INSERT INTO FUNCTION
|
||||
remoteSecure('remote.clickhouse.cloud:9440', 'imdb.actors', 'USER', 'PASSWORD', rand())
|
||||
SELECT * from imdb.actors
|
||||
```
|
||||
|
||||
## Globs in Addresses {#globs-in-addresses}
|
||||
|
||||
Patterns in curly brackets `{ }` are used to generate a set of shards and to specify replicas. If there are multiple pairs of curly brackets, then the direct product of the corresponding sets is generated.
|
||||
The following pattern types are supported.
|
||||
|
||||
@ -335,7 +335,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234
|
||||
|
||||
Поддерживаемые типы данных: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`.
|
||||
|
||||
Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem).
|
||||
Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions#hasany), [hasAll](../../../sql-reference/functions/array-functions#hasall).
|
||||
|
||||
**Примеры**
|
||||
|
||||
|
||||
@ -14,6 +14,15 @@
|
||||
- `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
|
||||
- `MUTATE_PART` — изменение куска.
|
||||
- `MOVE_PART` — перемещение куска между дисками.
|
||||
- `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — Причина события с типом `MERGE_PARTS`. Может принимать одно из следующих значений:
|
||||
- `NOT_A_MERGE` — событие имеет тип иной, чем `MERGE_PARTS`.
|
||||
- `REGULAR_MERGE` — обычное слияние.
|
||||
- `TTL_DELETE_MERGE` — очистка истекших данных.
|
||||
- `TTL_RECOMPRESS_MERGE` — переупаковка куска.
|
||||
- `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Алгоритм слияния для события с типом `MERGE_PARTS`. Может принимать одно из следующих значений:
|
||||
- `UNDECIDED`
|
||||
- `HORIZONTAL`
|
||||
- `VERTICAL`
|
||||
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата события.
|
||||
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время события.
|
||||
- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время события с точностью до микросекунд.
|
||||
@ -46,6 +55,8 @@ Row 1:
|
||||
──────
|
||||
query_id: 983ad9c7-28d5-4ae1-844e-603116b7de31
|
||||
event_type: NewPart
|
||||
merge_reason: NotAMerge
|
||||
merge_algorithm: Undecided
|
||||
event_date: 2021-02-02
|
||||
event_time: 2021-02-02 11:14:28
|
||||
event_time_microseconds: 2021-02-02 11:14:28.861919
|
||||
|
||||
@ -83,3 +83,34 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## translate(s, from, to)
|
||||
|
||||
Данная функция заменяет символы в строке ‘s’ в соответствии с поэлементным отображением определяемым строками ‘from’ и ‘to’. ‘from’ и ‘to’ должны быть корректными ASCII строками одного размера. Не ASCII символы в оригинальной строке не изменяются.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT translate('Hello, World!', 'delor', 'DELOR') AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ HELLO, WORLD! │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
## translateUTF8(string, from, to)
|
||||
|
||||
Аналогично предыдущей функции, но работает со строками, состоящими из UTF-8 символов. ‘from’ и ‘to’ должны быть корректными UTF-8 строками одного размера.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT translateUTF8('Hélló, Wórld¡', 'óé¡', 'oe!') AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ Hello, World! │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
@ -106,7 +106,9 @@ void Client::processError(const String & query) const
|
||||
std::vector<String> Client::loadWarningMessages()
|
||||
{
|
||||
std::vector<String> messages;
|
||||
connection->sendQuery(connection_parameters.timeouts, "SELECT message FROM system.warnings", "" /* query_id */,
|
||||
connection->sendQuery(connection_parameters.timeouts,
|
||||
"SELECT * FROM viewIfPermitted(SELECT message FROM system.warnings ELSE null('message String'))",
|
||||
"" /* query_id */,
|
||||
QueryProcessingStage::Complete,
|
||||
&global_context->getSettingsRef(),
|
||||
&global_context->getClientInfo(), false, {});
|
||||
|
||||
@ -20,7 +20,7 @@ public:
|
||||
|
||||
bool operator<=(const WrappingUInt32 & other) const
|
||||
{
|
||||
const UInt32 HALF = 1 << 31;
|
||||
const UInt32 HALF = static_cast<UInt32>(1) << 31;
|
||||
return (value <= other.value && other.value - value < HALF)
|
||||
|| (value > other.value && value - other.value > HALF);
|
||||
}
|
||||
|
||||
@ -604,9 +604,23 @@
|
||||
if this setting is true the user B will see all rows, and if this setting is false the user B will see no rows.
|
||||
By default this setting is false for compatibility with earlier access configurations. -->
|
||||
<users_without_row_policies_can_read_rows>false</users_without_row_policies_can_read_rows>
|
||||
|
||||
<!-- By default, for backward compatibility ON CLUSTER queries ignore CLUSTER grant,
|
||||
however you can change this behaviour by setting this to true -->
|
||||
<on_cluster_queries_require_cluster_grant>false</on_cluster_queries_require_cluster_grant>
|
||||
|
||||
<!-- By default, for backward compatibility "SELECT * FROM system.<table>" doesn't require any grants and can be executed
|
||||
by any user. You can change this behaviour by setting this to true.
|
||||
If it's set to true then this query requires "GRANT SELECT ON system.<table>" just like as for non-system tables.
|
||||
Exceptions: a few system tables ("tables", "columns", "databases", and some constant tables like "one", "contributors")
|
||||
are still accessible for everyone; and if there is a SHOW privilege (e.g. "SHOW USERS") granted the corresponding system
|
||||
table (i.e. "system.users") will be accessible. -->
|
||||
<select_from_system_db_requires_grant>false</select_from_system_db_requires_grant>
|
||||
|
||||
<!-- By default, for backward compatibility "SELECT * FROM information_schema.<table>" doesn't require any grants and can be
|
||||
executed by any user. You can change this behaviour by setting this to true.
|
||||
If it's set to true then this query requires "GRANT SELECT ON information_schema.<table>" just like as for ordinary tables. -->
|
||||
<select_from_information_schema_requires_grant>false</select_from_information_schema_requires_grant>
|
||||
</access_control_improvements>
|
||||
|
||||
<!-- Default profile of settings. -->
|
||||
|
||||
@ -101,7 +101,7 @@ public:
|
||||
registered_prefixes = prefixes_;
|
||||
}
|
||||
|
||||
bool isSettingNameAllowed(const std::string_view & setting_name) const
|
||||
bool isSettingNameAllowed(std::string_view setting_name) const
|
||||
{
|
||||
if (Settings::hasBuiltin(setting_name))
|
||||
return true;
|
||||
@ -116,7 +116,7 @@ public:
|
||||
return false;
|
||||
}
|
||||
|
||||
void checkSettingNameIsAllowed(const std::string_view & setting_name) const
|
||||
void checkSettingNameIsAllowed(std::string_view setting_name) const
|
||||
{
|
||||
if (isSettingNameAllowed(setting_name))
|
||||
return;
|
||||
@ -165,13 +165,12 @@ void AccessControl::setUpFromMainConfig(const Poco::Util::AbstractConfiguration
|
||||
setNoPasswordAllowed(config_.getBool("allow_no_password", true));
|
||||
setPlaintextPasswordAllowed(config_.getBool("allow_plaintext_password", true));
|
||||
|
||||
setEnabledUsersWithoutRowPoliciesCanReadRows(config_.getBool(
|
||||
"access_control_improvements.users_without_row_policies_can_read_rows",
|
||||
false /* false because we need to be compatible with earlier access configurations */));
|
||||
|
||||
setOnClusterQueriesRequireClusterGrant(config_.getBool(
|
||||
"access_control_improvements.on_cluster_queries_require_cluster_grant",
|
||||
false /* false because we need to be compatible with earlier access configurations */));
|
||||
/// Optional improvements in access control system.
|
||||
/// The default values are false because we need to be compatible with earlier access configurations
|
||||
setEnabledUsersWithoutRowPoliciesCanReadRows(config_.getBool("access_control_improvements.users_without_row_policies_can_read_rows", false));
|
||||
setOnClusterQueriesRequireClusterGrant(config_.getBool("access_control_improvements.on_cluster_queries_require_cluster_grant", false));
|
||||
setSelectFromSystemDatabaseRequiresGrant(config_.getBool("access_control_improvements.select_from_system_db_requires_grant", false));
|
||||
setSelectFromInformationSchemaRequiresGrant(config_.getBool("access_control_improvements.select_from_information_schema_requires_grant", false));
|
||||
|
||||
addStoragesFromMainConfig(config_, config_path_, get_zookeeper_function_);
|
||||
}
|
||||
|
||||
@ -152,6 +152,12 @@ public:
|
||||
void setOnClusterQueriesRequireClusterGrant(bool enable) { on_cluster_queries_require_cluster_grant = enable; }
|
||||
bool doesOnClusterQueriesRequireClusterGrant() const { return on_cluster_queries_require_cluster_grant; }
|
||||
|
||||
void setSelectFromSystemDatabaseRequiresGrant(bool enable) { select_from_system_db_requires_grant = enable; }
|
||||
bool doesSelectFromSystemDatabaseRequireGrant() const { return select_from_system_db_requires_grant; }
|
||||
|
||||
void setSelectFromInformationSchemaRequiresGrant(bool enable) { select_from_information_schema_requires_grant = enable; }
|
||||
bool doesSelectFromInformationSchemaRequireGrant() const { return select_from_information_schema_requires_grant; }
|
||||
|
||||
std::shared_ptr<const ContextAccess> getContextAccess(
|
||||
const UUID & user_id,
|
||||
const std::vector<UUID> & current_roles,
|
||||
@ -215,6 +221,8 @@ private:
|
||||
std::atomic_bool allow_no_password = true;
|
||||
std::atomic_bool users_without_row_policies_can_read_rows = false;
|
||||
std::atomic_bool on_cluster_queries_require_cluster_grant = false;
|
||||
std::atomic_bool select_from_system_db_requires_grant = false;
|
||||
std::atomic_bool select_from_information_schema_requires_grant = false;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -252,7 +252,7 @@ public:
|
||||
}
|
||||
|
||||
template <typename ... Args>
|
||||
void grant(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
|
||||
void grant(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
|
||||
{
|
||||
auto & child = getChild(name);
|
||||
child.grant(flags_, subnames...);
|
||||
@ -279,7 +279,7 @@ public:
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
void revoke(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
|
||||
void revoke(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
|
||||
{
|
||||
auto & child = getChild(name);
|
||||
|
||||
@ -306,7 +306,7 @@ public:
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
bool isGranted(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames) const
|
||||
bool isGranted(const AccessFlags & flags_, std::string_view name, const Args &... subnames) const
|
||||
{
|
||||
AccessFlags flags_to_check = flags_ - min_flags_with_children;
|
||||
if (!flags_to_check)
|
||||
@ -388,11 +388,11 @@ public:
|
||||
return res;
|
||||
}
|
||||
|
||||
void modifyFlags(const ModifyFlagsFunction & function, bool & flags_added, bool & flags_removed)
|
||||
void modifyFlags(const ModifyFlagsFunction & function, bool grant_option, bool & flags_added, bool & flags_removed)
|
||||
{
|
||||
flags_added = false;
|
||||
flags_removed = false;
|
||||
modifyFlagsRec(function, flags_added, flags_removed);
|
||||
modifyFlagsRec(function, grant_option, flags_added, flags_removed);
|
||||
if (flags_added || flags_removed)
|
||||
optimizeTree();
|
||||
}
|
||||
@ -415,7 +415,7 @@ private:
|
||||
AccessFlags getAllGrantableFlags() const { return ::DB::getAllGrantableFlags(level); }
|
||||
AccessFlags getChildAllGrantableFlags() const { return ::DB::getAllGrantableFlags(static_cast<Level>(level + 1)); }
|
||||
|
||||
Node * tryGetChild(const std::string_view & name) const
|
||||
Node * tryGetChild(std::string_view name) const
|
||||
{
|
||||
if (!children)
|
||||
return nullptr;
|
||||
@ -425,7 +425,7 @@ private:
|
||||
return &it->second;
|
||||
}
|
||||
|
||||
Node & getChild(const std::string_view & name)
|
||||
Node & getChild(std::string_view name)
|
||||
{
|
||||
auto * child = tryGetChild(name);
|
||||
if (child)
|
||||
@ -669,11 +669,11 @@ private:
|
||||
}
|
||||
|
||||
template <typename ... ParentNames>
|
||||
void modifyFlagsRec(const ModifyFlagsFunction & function, bool & flags_added, bool & flags_removed, const ParentNames & ... parent_names)
|
||||
void modifyFlagsRec(const ModifyFlagsFunction & function, bool grant_option, bool & flags_added, bool & flags_removed, const ParentNames & ... parent_names)
|
||||
{
|
||||
auto invoke = [&function](const AccessFlags & flags_, const AccessFlags & min_flags_with_children_, const AccessFlags & max_flags_with_children_, std::string_view database_ = {}, std::string_view table_ = {}, std::string_view column_ = {}) -> AccessFlags
|
||||
auto invoke = [function, grant_option](const AccessFlags & flags_, const AccessFlags & min_flags_with_children_, const AccessFlags & max_flags_with_children_, std::string_view database_ = {}, std::string_view table_ = {}, std::string_view column_ = {}) -> AccessFlags
|
||||
{
|
||||
return function(flags_, min_flags_with_children_, max_flags_with_children_, database_, table_, column_);
|
||||
return function(flags_, min_flags_with_children_, max_flags_with_children_, database_, table_, column_, grant_option);
|
||||
};
|
||||
|
||||
if constexpr (sizeof...(ParentNames) < 3)
|
||||
@ -683,7 +683,7 @@ private:
|
||||
for (auto & child : *children | boost::adaptors::map_values)
|
||||
{
|
||||
const String & child_name = *child.node_name;
|
||||
child.modifyFlagsRec(function, flags_added, flags_removed, parent_names..., child_name);
|
||||
child.modifyFlagsRec(function, grant_option, flags_added, flags_removed, parent_names..., child_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -819,20 +819,20 @@ void AccessRights::grantImpl(const AccessRightsElements & elements)
|
||||
}
|
||||
|
||||
void AccessRights::grant(const AccessFlags & flags) { grantImpl<false>(flags); }
|
||||
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl<false>(flags, database); }
|
||||
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<false>(flags, database, table); }
|
||||
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<false>(flags, database, table, column); }
|
||||
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::grant(const AccessFlags & flags, std::string_view database) { grantImpl<false>(flags, database); }
|
||||
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<false>(flags, database, table); }
|
||||
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<false>(flags, database, table, column); }
|
||||
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::grant(const AccessRightsElement & element) { grantImpl<false>(element); }
|
||||
void AccessRights::grant(const AccessRightsElements & elements) { grantImpl<false>(elements); }
|
||||
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl<true>(flags); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database) { grantImpl<true>(flags, database); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<true>(flags, database, table); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<true>(flags, database, table, column); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database) { grantImpl<true>(flags, database); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<true>(flags, database, table); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<true>(flags, database, table, column); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::grantWithGrantOption(const AccessRightsElement & element) { grantImpl<true>(element); }
|
||||
void AccessRights::grantWithGrantOption(const AccessRightsElements & elements) { grantImpl<true>(elements); }
|
||||
|
||||
@ -892,20 +892,20 @@ void AccessRights::revokeImpl(const AccessRightsElements & elements)
|
||||
}
|
||||
|
||||
void AccessRights::revoke(const AccessFlags & flags) { revokeImpl<false>(flags); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl<false>(flags, database); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<false>(flags, database, table); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<false>(flags, database, table, column); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, std::string_view database) { revokeImpl<false>(flags, database); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<false>(flags, database, table); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<false>(flags, database, table, column); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::revoke(const AccessRightsElement & element) { revokeImpl<false>(element); }
|
||||
void AccessRights::revoke(const AccessRightsElements & elements) { revokeImpl<false>(elements); }
|
||||
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags) { revokeImpl<true>(flags); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database) { revokeImpl<true>(flags, database); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<true>(flags, database, table); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<true>(flags, database, table, column); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database) { revokeImpl<true>(flags, database); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<true>(flags, database, table); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<true>(flags, database, table, column); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::revokeGrantOption(const AccessRightsElement & element) { revokeImpl<true>(element); }
|
||||
void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { revokeImpl<true>(elements); }
|
||||
|
||||
@ -984,20 +984,20 @@ bool AccessRights::isGrantedImpl(const AccessRightsElements & elements) const
|
||||
}
|
||||
|
||||
bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl<false>(flags); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<false>(flags, database); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<false>(flags, database, table); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<false>(flags, database, table, column); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<false>(flags, database); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<false>(flags, database, table); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<false>(flags, database, table, column); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
|
||||
bool AccessRights::isGranted(const AccessRightsElement & element) const { return isGrantedImpl<false>(element); }
|
||||
bool AccessRights::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl<false>(elements); }
|
||||
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl<true>(flags); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<true>(flags, database); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<true>(flags, database, table); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<true>(flags, database, table, column); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<true>(flags, database); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<true>(flags, database, table); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<true>(flags, database, table, column); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
|
||||
bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl<true>(element); }
|
||||
bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl<true>(elements); }
|
||||
|
||||
@ -1062,24 +1062,21 @@ void AccessRights::modifyFlags(const ModifyFlagsFunction & function)
|
||||
{
|
||||
if (!root)
|
||||
return;
|
||||
|
||||
bool flags_added, flags_removed;
|
||||
root->modifyFlags(function, flags_added, flags_removed);
|
||||
root->modifyFlags(function, false, flags_added, flags_removed);
|
||||
if (flags_removed && root_with_grant_option)
|
||||
root_with_grant_option->makeIntersection(*root);
|
||||
}
|
||||
|
||||
|
||||
void AccessRights::modifyFlagsWithGrantOption(const ModifyFlagsFunction & function)
|
||||
{
|
||||
if (!root_with_grant_option)
|
||||
return;
|
||||
bool flags_added, flags_removed;
|
||||
root_with_grant_option->modifyFlags(function, flags_added, flags_removed);
|
||||
if (flags_added)
|
||||
if (root_with_grant_option)
|
||||
{
|
||||
if (!root)
|
||||
root = std::make_unique<Node>();
|
||||
root->makeUnion(*root_with_grant_option);
|
||||
root_with_grant_option->modifyFlags(function, true, flags_added, flags_removed);
|
||||
if (flags_added)
|
||||
{
|
||||
if (!root)
|
||||
root = std::make_unique<Node>();
|
||||
root->makeUnion(*root_with_grant_option);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -39,59 +39,59 @@ public:
|
||||
/// Grants access on a specified database/table/column.
|
||||
/// Does nothing if the specified access has been already granted.
|
||||
void grant(const AccessFlags & flags);
|
||||
void grant(const AccessFlags & flags, const std::string_view & database);
|
||||
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
|
||||
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
|
||||
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
|
||||
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
|
||||
void grant(const AccessFlags & flags, std::string_view database);
|
||||
void grant(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
void grant(const AccessRightsElement & element);
|
||||
void grant(const AccessRightsElements & elements);
|
||||
|
||||
void grantWithGrantOption(const AccessFlags & flags);
|
||||
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database);
|
||||
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
|
||||
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
|
||||
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
|
||||
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
void grantWithGrantOption(const AccessRightsElement & element);
|
||||
void grantWithGrantOption(const AccessRightsElements & elements);
|
||||
|
||||
/// Revokes a specified access granted earlier on a specified database/table/column.
|
||||
/// For example, revoke(AccessType::ALL) revokes all grants at all, just like clear();
|
||||
void revoke(const AccessFlags & flags);
|
||||
void revoke(const AccessFlags & flags, const std::string_view & database);
|
||||
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
|
||||
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
|
||||
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
|
||||
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
|
||||
void revoke(const AccessFlags & flags, std::string_view database);
|
||||
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
void revoke(const AccessRightsElement & element);
|
||||
void revoke(const AccessRightsElements & elements);
|
||||
|
||||
void revokeGrantOption(const AccessFlags & flags);
|
||||
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database);
|
||||
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
|
||||
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
|
||||
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
|
||||
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
void revokeGrantOption(const AccessRightsElement & element);
|
||||
void revokeGrantOption(const AccessRightsElements & elements);
|
||||
|
||||
/// Whether a specified access granted.
|
||||
bool isGranted(const AccessFlags & flags) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool isGranted(const AccessRightsElement & element) const;
|
||||
bool isGranted(const AccessRightsElements & elements) const;
|
||||
|
||||
bool hasGrantOption(const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const AccessRightsElements & elements) const;
|
||||
|
||||
@ -107,11 +107,11 @@ public:
|
||||
const AccessFlags & flags,
|
||||
const AccessFlags & min_flags_with_children,
|
||||
const AccessFlags & max_flags_with_children,
|
||||
const std::string_view & database,
|
||||
const std::string_view & table,
|
||||
const std::string_view & column)>;
|
||||
std::string_view database,
|
||||
std::string_view table,
|
||||
std::string_view column,
|
||||
bool grant_option)>;
|
||||
void modifyFlags(const ModifyFlagsFunction & function);
|
||||
void modifyFlagsWithGrantOption(const ModifyFlagsFunction & function);
|
||||
|
||||
friend bool operator ==(const AccessRights & left, const AccessRights & right);
|
||||
friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); }
|
||||
|
||||
@ -26,17 +26,17 @@ namespace
|
||||
return (Util::stringToDigest(password) == password_plaintext);
|
||||
}
|
||||
|
||||
bool checkPasswordDoubleSHA1(const std::string_view & password, const Digest & password_double_sha1)
|
||||
bool checkPasswordDoubleSHA1(std::string_view password, const Digest & password_double_sha1)
|
||||
{
|
||||
return (Util::encodeDoubleSHA1(password) == password_double_sha1);
|
||||
}
|
||||
|
||||
bool checkPasswordSHA256(const std::string_view & password, const Digest & password_sha256, const String & salt)
|
||||
bool checkPasswordSHA256(std::string_view password, const Digest & password_sha256, const String & salt)
|
||||
{
|
||||
return Util::encodeSHA256(String(password).append(salt)) == password_sha256;
|
||||
}
|
||||
|
||||
bool checkPasswordDoubleSHA1MySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_double_sha1)
|
||||
bool checkPasswordDoubleSHA1MySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_double_sha1)
|
||||
{
|
||||
/// scrambled_password = SHA1(password) XOR SHA1(scramble <concat> SHA1(SHA1(password)))
|
||||
|
||||
@ -61,7 +61,7 @@ namespace
|
||||
return calculated_password_double_sha1 == password_double_sha1;
|
||||
}
|
||||
|
||||
bool checkPasswordPlainTextMySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_plaintext)
|
||||
bool checkPasswordPlainTextMySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_plaintext)
|
||||
{
|
||||
return checkPasswordDoubleSHA1MySQL(scramble, scrambled_password, Util::encodeDoubleSHA1(password_plaintext));
|
||||
}
|
||||
|
||||
@ -35,7 +35,7 @@ namespace
|
||||
return access_type_to_flags_mapping[static_cast<size_t>(type)];
|
||||
}
|
||||
|
||||
Flags keywordToFlags(const std::string_view & keyword) const
|
||||
Flags keywordToFlags(std::string_view keyword) const
|
||||
{
|
||||
auto it = keyword_to_flags_map.find(keyword);
|
||||
if (it == keyword_to_flags_map.end())
|
||||
@ -142,14 +142,14 @@ namespace
|
||||
}
|
||||
};
|
||||
|
||||
static String replaceUnderscoreWithSpace(const std::string_view & str)
|
||||
static String replaceUnderscoreWithSpace(std::string_view str)
|
||||
{
|
||||
String res{str};
|
||||
boost::replace_all(res, "_", " ");
|
||||
return res;
|
||||
}
|
||||
|
||||
static Strings splitAliases(const std::string_view & str)
|
||||
static Strings splitAliases(std::string_view str)
|
||||
{
|
||||
Strings aliases;
|
||||
boost::split(aliases, str, boost::is_any_of(","));
|
||||
@ -160,10 +160,10 @@ namespace
|
||||
|
||||
static void makeNode(
|
||||
AccessType access_type,
|
||||
const std::string_view & name,
|
||||
const std::string_view & aliases,
|
||||
std::string_view name,
|
||||
std::string_view aliases,
|
||||
NodeType node_type,
|
||||
const std::string_view & parent_group_name,
|
||||
std::string_view parent_group_name,
|
||||
std::unordered_map<std::string_view, Node *> & nodes,
|
||||
std::unordered_map<std::string_view, NodePtr> & owned_nodes,
|
||||
size_t & next_flag)
|
||||
@ -353,7 +353,7 @@ namespace
|
||||
|
||||
|
||||
AccessFlags::AccessFlags(AccessType type) : flags(Helper::instance().accessTypeToFlags(type)) {}
|
||||
AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
|
||||
AccessFlags::AccessFlags(std::string_view keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
|
||||
AccessFlags::AccessFlags(const std::vector<std::string_view> & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
|
||||
AccessFlags::AccessFlags(const Strings & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
|
||||
String AccessFlags::toString() const { return Helper::instance().flagsToString(flags); }
|
||||
|
||||
@ -21,7 +21,7 @@ public:
|
||||
AccessFlags() = default;
|
||||
|
||||
/// Constructs from a string like "SELECT".
|
||||
AccessFlags(const std::string_view & keyword); /// NOLINT
|
||||
AccessFlags(std::string_view keyword); /// NOLINT
|
||||
|
||||
/// Constructs from a list of strings like "SELECT, UPDATE, INSERT".
|
||||
AccessFlags(const std::vector<std::string_view> & keywords); /// NOLINT
|
||||
|
||||
@ -81,7 +81,7 @@ namespace
|
||||
}
|
||||
|
||||
bool need_comma = false;
|
||||
for (const std::string_view & keyword : keywords)
|
||||
for (std::string_view keyword : keywords)
|
||||
{
|
||||
if (need_comma)
|
||||
result.append(", ");
|
||||
@ -145,18 +145,18 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_)
|
||||
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_)
|
||||
: access_flags(access_flags_), database(database_), any_database(false)
|
||||
{
|
||||
}
|
||||
|
||||
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_)
|
||||
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_)
|
||||
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false)
|
||||
{
|
||||
}
|
||||
|
||||
AccessRightsElement::AccessRightsElement(
|
||||
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_)
|
||||
AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_)
|
||||
: access_flags(access_flags_)
|
||||
, database(database_)
|
||||
, table(table_)
|
||||
@ -169,8 +169,8 @@ AccessRightsElement::AccessRightsElement(
|
||||
|
||||
AccessRightsElement::AccessRightsElement(
|
||||
AccessFlags access_flags_,
|
||||
const std::string_view & database_,
|
||||
const std::string_view & table_,
|
||||
std::string_view database_,
|
||||
std::string_view table_,
|
||||
const std::vector<std::string_view> & columns_)
|
||||
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false), any_column(false)
|
||||
{
|
||||
@ -180,7 +180,7 @@ AccessRightsElement::AccessRightsElement(
|
||||
}
|
||||
|
||||
AccessRightsElement::AccessRightsElement(
|
||||
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_)
|
||||
AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_)
|
||||
: access_flags(access_flags_)
|
||||
, database(database_)
|
||||
, table(table_)
|
||||
|
||||
@ -28,19 +28,19 @@ struct AccessRightsElement
|
||||
|
||||
explicit AccessRightsElement(AccessFlags access_flags_) : access_flags(access_flags_) {}
|
||||
|
||||
AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_);
|
||||
AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_);
|
||||
AccessRightsElement(AccessFlags access_flags_, std::string_view database_);
|
||||
AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_);
|
||||
AccessRightsElement(
|
||||
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_);
|
||||
AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_);
|
||||
|
||||
AccessRightsElement(
|
||||
AccessFlags access_flags_,
|
||||
const std::string_view & database_,
|
||||
const std::string_view & table_,
|
||||
std::string_view database_,
|
||||
std::string_view table_,
|
||||
const std::vector<std::string_view> & columns_);
|
||||
|
||||
AccessRightsElement(
|
||||
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_);
|
||||
AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_);
|
||||
|
||||
bool empty() const { return !access_flags || (!any_column && columns.empty()); }
|
||||
|
||||
|
||||
@ -35,7 +35,7 @@ namespace
|
||||
#undef ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING
|
||||
}
|
||||
|
||||
void addToMapping(AccessType type, const std::string_view & str)
|
||||
void addToMapping(AccessType type, std::string_view str)
|
||||
{
|
||||
String str2{str};
|
||||
boost::replace_all(str2, "_", " ");
|
||||
|
||||
@ -71,7 +71,7 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty
|
||||
}
|
||||
|
||||
|
||||
AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::string_view & text [[maybe_unused]])
|
||||
AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_view text [[maybe_unused]])
|
||||
{
|
||||
#if USE_SSL
|
||||
Digest hash;
|
||||
@ -86,7 +86,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::str
|
||||
}
|
||||
|
||||
|
||||
AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(const std::string_view & text)
|
||||
AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(std::string_view text)
|
||||
{
|
||||
Poco::SHA1Engine engine;
|
||||
engine.update(text.data(), text.size());
|
||||
|
||||
@ -96,11 +96,11 @@ public:
|
||||
|
||||
struct Util
|
||||
{
|
||||
static Digest stringToDigest(const std::string_view & text) { return Digest(text.data(), text.data() + text.size()); }
|
||||
static Digest encodeSHA256(const std::string_view & text);
|
||||
static Digest encodeSHA1(const std::string_view & text);
|
||||
static Digest stringToDigest(std::string_view text) { return Digest(text.data(), text.data() + text.size()); }
|
||||
static Digest encodeSHA256(std::string_view text);
|
||||
static Digest encodeSHA1(std::string_view text);
|
||||
static Digest encodeSHA1(const Digest & text) { return encodeSHA1(std::string_view{reinterpret_cast<const char *>(text.data()), text.size()}); }
|
||||
static Digest encodeDoubleSHA1(const std::string_view & text) { return encodeSHA1(encodeSHA1(text)); }
|
||||
static Digest encodeDoubleSHA1(std::string_view text) { return encodeSHA1(encodeSHA1(text)); }
|
||||
static Digest encodeDoubleSHA1(const Digest & text) { return encodeSHA1(encodeSHA1(text)); }
|
||||
};
|
||||
|
||||
|
||||
@ -44,9 +44,17 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
AccessRights addImplicitAccessRights(const AccessRights & access)
|
||||
AccessRights addImplicitAccessRights(const AccessRights & access, const AccessControl & access_control)
|
||||
{
|
||||
auto modifier = [&](const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, const std::string_view & database, const std::string_view & table, const std::string_view & column) -> AccessFlags
|
||||
AccessFlags max_flags;
|
||||
|
||||
auto modifier = [&](const AccessFlags & flags,
|
||||
const AccessFlags & min_flags_with_children,
|
||||
const AccessFlags & max_flags_with_children,
|
||||
std::string_view database,
|
||||
std::string_view table,
|
||||
std::string_view column,
|
||||
bool /* grant_option */) -> AccessFlags
|
||||
{
|
||||
size_t level = !database.empty() + !table.empty() + !column.empty();
|
||||
AccessFlags res = flags;
|
||||
@ -115,17 +123,80 @@ namespace
|
||||
res |= show_databases;
|
||||
}
|
||||
|
||||
max_flags |= res;
|
||||
|
||||
return res;
|
||||
};
|
||||
|
||||
AccessRights res = access;
|
||||
res.modifyFlags(modifier);
|
||||
res.modifyFlagsWithGrantOption(modifier);
|
||||
|
||||
/// Anyone has access to the "system" and "information_schema" database.
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA);
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE);
|
||||
/// If "select_from_system_db_requires_grant" is enabled we provide implicit grants only for a few tables in the system database.
|
||||
if (access_control.doesSelectFromSystemDatabaseRequireGrant())
|
||||
{
|
||||
const char * always_accessible_tables[] = {
|
||||
/// Constant tables
|
||||
"one",
|
||||
|
||||
/// "numbers", "numbers_mt", "zeros", "zeros_mt" were excluded because they can generate lots of values and
|
||||
/// that can decrease performance in some cases.
|
||||
|
||||
"contributors",
|
||||
"licenses",
|
||||
"time_zones",
|
||||
"collations",
|
||||
|
||||
"formats",
|
||||
"privileges",
|
||||
"data_type_families",
|
||||
"table_engines",
|
||||
"table_functions",
|
||||
"aggregate_function_combinators",
|
||||
|
||||
"functions", /// Can contain user-defined functions
|
||||
|
||||
/// The following tables hide some rows if the current user doesn't have corresponding SHOW privileges.
|
||||
"databases",
|
||||
"tables",
|
||||
"columns",
|
||||
|
||||
/// Specific to the current session
|
||||
"settings",
|
||||
"current_roles",
|
||||
"enabled_roles",
|
||||
"quota_usage"
|
||||
};
|
||||
|
||||
for (const auto * table_name : always_accessible_tables)
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, table_name);
|
||||
|
||||
if (max_flags.contains(AccessType::SHOW_USERS))
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "users");
|
||||
|
||||
if (max_flags.contains(AccessType::SHOW_ROLES))
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "roles");
|
||||
|
||||
if (max_flags.contains(AccessType::SHOW_ROW_POLICIES))
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "row_policies");
|
||||
|
||||
if (max_flags.contains(AccessType::SHOW_SETTINGS_PROFILES))
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "settings_profiles");
|
||||
|
||||
if (max_flags.contains(AccessType::SHOW_QUOTAS))
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "quotas");
|
||||
}
|
||||
else
|
||||
{
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
|
||||
}
|
||||
|
||||
/// If "select_from_information_schema_requires_grant" is enabled we don't provide implicit grants for the information_schema database.
|
||||
if (!access_control.doesSelectFromInformationSchemaRequireGrant())
|
||||
{
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA);
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -141,7 +212,7 @@ namespace
|
||||
std::string_view getDatabase() { return {}; }
|
||||
|
||||
template <typename... OtherArgs>
|
||||
std::string_view getDatabase(const std::string_view & arg1, const OtherArgs &...) { return arg1; }
|
||||
std::string_view getDatabase(std::string_view arg1, const OtherArgs &...) { return arg1; }
|
||||
}
|
||||
|
||||
|
||||
@ -247,7 +318,7 @@ void ContextAccess::setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> &
|
||||
void ContextAccess::calculateAccessRights() const
|
||||
{
|
||||
access = std::make_shared<AccessRights>(mixAccessRightsFromUserAndRoles(*user, *roles_info));
|
||||
access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*access));
|
||||
access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*access, *access_control));
|
||||
|
||||
if (trace_log)
|
||||
{
|
||||
@ -342,7 +413,7 @@ std::shared_ptr<const ContextAccess> ContextAccess::getFullAccess()
|
||||
auto full_access = std::shared_ptr<ContextAccess>(new ContextAccess);
|
||||
full_access->is_full_access = true;
|
||||
full_access->access = std::make_shared<AccessRights>(AccessRights::getFullAccess());
|
||||
full_access->access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*full_access->access));
|
||||
full_access->access_with_implicit = full_access->access;
|
||||
return full_access;
|
||||
}();
|
||||
return res;
|
||||
@ -413,7 +484,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
};
|
||||
|
||||
if (is_full_access)
|
||||
return access_granted();
|
||||
return true;
|
||||
|
||||
if (user_was_dropped)
|
||||
return access_denied("User has been dropped", ErrorCodes::UNKNOWN_USER);
|
||||
@ -422,7 +493,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
flags &= ~AccessType::CLUSTER;
|
||||
|
||||
if (!flags)
|
||||
return access_granted();
|
||||
return true;
|
||||
|
||||
/// Access to temporary tables is controlled in an unusual way, not like normal tables.
|
||||
/// Creating of temporary tables is controlled by AccessType::CREATE_TEMPORARY_TABLES grant,
|
||||
@ -519,7 +590,7 @@ bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const
|
||||
bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(flags, database.empty() ? params.current_database : database, args...);
|
||||
}
|
||||
@ -564,38 +635,38 @@ bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
|
||||
}
|
||||
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl<false, false>(flags); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, false>(flags, database); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, false>(flags, database, table); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(flags, database); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(flags, database, table); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl<false, false>(element); }
|
||||
bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(elements); }
|
||||
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return checkAccessImpl<false, true>(flags); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, true>(flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, true>(flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return checkAccessImpl<false, true>(element); }
|
||||
bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(elements); }
|
||||
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<true, false>(flags); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, false>(flags, database); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, false>(flags, database, table); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, false>(flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(flags, database); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(flags, database, table); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<true, false>(element); }
|
||||
void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<true, false>(elements); }
|
||||
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<true, true>(flags); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, true>(flags, database); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, true>(flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, true>(flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(flags, database); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<true, true>(element); }
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true, true>(elements); }
|
||||
|
||||
|
||||
@ -101,40 +101,40 @@ public:
|
||||
/// Checks if a specified access is granted, and throws an exception if not.
|
||||
/// Empty database means the current database.
|
||||
void checkAccess(const AccessFlags & flags) const;
|
||||
void checkAccess(const AccessFlags & flags, const std::string_view & database) const;
|
||||
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkAccess(const AccessRightsElement & element) const;
|
||||
void checkAccess(const AccessRightsElements & elements) const;
|
||||
|
||||
void checkGrantOption(const AccessFlags & flags) const;
|
||||
void checkGrantOption(const AccessFlags & flags, const std::string_view & database) const;
|
||||
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkGrantOption(const AccessRightsElement & element) const;
|
||||
void checkGrantOption(const AccessRightsElements & elements) const;
|
||||
|
||||
/// Checks if a specified access is granted, and returns false if not.
|
||||
/// Empty database means the current database.
|
||||
bool isGranted(const AccessFlags & flags) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool isGranted(const AccessRightsElement & element) const;
|
||||
bool isGranted(const AccessRightsElements & elements) const;
|
||||
|
||||
bool hasGrantOption(const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const AccessRightsElements & elements) const;
|
||||
|
||||
@ -180,7 +180,7 @@ private:
|
||||
bool checkAccessImpl(const AccessFlags & flags) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const;
|
||||
bool checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessRightsElement & element) const;
|
||||
|
||||
@ -36,12 +36,12 @@ void SettingsConstraints::clear()
|
||||
}
|
||||
|
||||
|
||||
void SettingsConstraints::setMinValue(const std::string_view & setting_name, const Field & min_value)
|
||||
void SettingsConstraints::setMinValue(std::string_view setting_name, const Field & min_value)
|
||||
{
|
||||
getConstraintRef(setting_name).min_value = Settings::castValueUtil(setting_name, min_value);
|
||||
}
|
||||
|
||||
Field SettingsConstraints::getMinValue(const std::string_view & setting_name) const
|
||||
Field SettingsConstraints::getMinValue(std::string_view setting_name) const
|
||||
{
|
||||
const auto * ptr = tryGetConstraint(setting_name);
|
||||
if (ptr)
|
||||
@ -51,12 +51,12 @@ Field SettingsConstraints::getMinValue(const std::string_view & setting_name) co
|
||||
}
|
||||
|
||||
|
||||
void SettingsConstraints::setMaxValue(const std::string_view & setting_name, const Field & max_value)
|
||||
void SettingsConstraints::setMaxValue(std::string_view setting_name, const Field & max_value)
|
||||
{
|
||||
getConstraintRef(setting_name).max_value = Settings::castValueUtil(setting_name, max_value);
|
||||
}
|
||||
|
||||
Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) const
|
||||
Field SettingsConstraints::getMaxValue(std::string_view setting_name) const
|
||||
{
|
||||
const auto * ptr = tryGetConstraint(setting_name);
|
||||
if (ptr)
|
||||
@ -66,12 +66,12 @@ Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) co
|
||||
}
|
||||
|
||||
|
||||
void SettingsConstraints::setReadOnly(const std::string_view & setting_name, bool read_only)
|
||||
void SettingsConstraints::setReadOnly(std::string_view setting_name, bool read_only)
|
||||
{
|
||||
getConstraintRef(setting_name).read_only = read_only;
|
||||
}
|
||||
|
||||
bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) const
|
||||
bool SettingsConstraints::isReadOnly(std::string_view setting_name) const
|
||||
{
|
||||
const auto * ptr = tryGetConstraint(setting_name);
|
||||
if (ptr)
|
||||
@ -81,7 +81,7 @@ bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) cons
|
||||
}
|
||||
|
||||
|
||||
void SettingsConstraints::set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only)
|
||||
void SettingsConstraints::set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only)
|
||||
{
|
||||
auto & ref = getConstraintRef(setting_name);
|
||||
ref.min_value = Settings::castValueUtil(setting_name, min_value);
|
||||
@ -89,7 +89,7 @@ void SettingsConstraints::set(const std::string_view & setting_name, const Field
|
||||
ref.read_only = read_only;
|
||||
}
|
||||
|
||||
void SettingsConstraints::get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const
|
||||
void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const
|
||||
{
|
||||
const auto * ptr = tryGetConstraint(setting_name);
|
||||
if (ptr)
|
||||
@ -318,7 +318,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh
|
||||
}
|
||||
|
||||
|
||||
SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const std::string_view & setting_name)
|
||||
SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(std::string_view setting_name)
|
||||
{
|
||||
auto it = constraints.find(setting_name);
|
||||
if (it == constraints.end())
|
||||
@ -331,7 +331,7 @@ SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const st
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(const std::string_view & setting_name) const
|
||||
const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(std::string_view setting_name) const
|
||||
{
|
||||
auto it = constraints.find(setting_name);
|
||||
if (it == constraints.end())
|
||||
|
||||
@ -61,17 +61,17 @@ public:
|
||||
void clear();
|
||||
bool empty() const { return constraints.empty(); }
|
||||
|
||||
void setMinValue(const std::string_view & setting_name, const Field & min_value);
|
||||
Field getMinValue(const std::string_view & setting_name) const;
|
||||
void setMinValue(std::string_view setting_name, const Field & min_value);
|
||||
Field getMinValue(std::string_view setting_name) const;
|
||||
|
||||
void setMaxValue(const std::string_view & setting_name, const Field & max_value);
|
||||
Field getMaxValue(const std::string_view & setting_name) const;
|
||||
void setMaxValue(std::string_view setting_name, const Field & max_value);
|
||||
Field getMaxValue(std::string_view setting_name) const;
|
||||
|
||||
void setReadOnly(const std::string_view & setting_name, bool read_only);
|
||||
bool isReadOnly(const std::string_view & setting_name) const;
|
||||
void setReadOnly(std::string_view setting_name, bool read_only);
|
||||
bool isReadOnly(std::string_view setting_name) const;
|
||||
|
||||
void set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only);
|
||||
void get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const;
|
||||
void set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only);
|
||||
void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const;
|
||||
|
||||
void merge(const SettingsConstraints & other);
|
||||
|
||||
@ -105,8 +105,8 @@ private:
|
||||
};
|
||||
bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const;
|
||||
|
||||
Constraint & getConstraintRef(const std::string_view & setting_name);
|
||||
const Constraint * tryGetConstraint(const std::string_view & setting_name) const;
|
||||
Constraint & getConstraintRef(std::string_view setting_name);
|
||||
const Constraint * tryGetConstraint(std::string_view setting_name) const;
|
||||
|
||||
std::unordered_map<std::string_view, Constraint> constraints;
|
||||
const AccessControl * access_control = nullptr;
|
||||
|
||||
@ -107,7 +107,7 @@ UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & c
|
||||
{
|
||||
if (async)
|
||||
{
|
||||
query_scope.emplace(context_in_use);
|
||||
query_scope.emplace(mutable_context);
|
||||
setThreadName("BackupWorker");
|
||||
}
|
||||
|
||||
|
||||
@ -404,11 +404,11 @@ dbms_target_link_libraries (
|
||||
clickhouse_parsers
|
||||
ch_contrib::lz4
|
||||
Poco::JSON
|
||||
Poco::MongoDB
|
||||
string_utils
|
||||
PUBLIC
|
||||
boost::system
|
||||
clickhouse_common_io
|
||||
Poco::MongoDB
|
||||
)
|
||||
|
||||
if (TARGET ch::mysqlxx)
|
||||
|
||||
@ -601,6 +601,7 @@ void ClientBase::initLogsOutputStream()
|
||||
{
|
||||
WriteBuffer * wb = out_logs_buf.get();
|
||||
|
||||
bool color_logs = false;
|
||||
if (!out_logs_buf)
|
||||
{
|
||||
if (server_logs_file.empty())
|
||||
@ -608,11 +609,13 @@ void ClientBase::initLogsOutputStream()
|
||||
/// Use stderr by default
|
||||
out_logs_buf = std::make_unique<WriteBufferFromFileDescriptor>(STDERR_FILENO);
|
||||
wb = out_logs_buf.get();
|
||||
color_logs = stderr_is_a_tty;
|
||||
}
|
||||
else if (server_logs_file == "-")
|
||||
{
|
||||
/// Use stdout if --server_logs_file=- specified
|
||||
wb = &std_out;
|
||||
color_logs = stdout_is_a_tty;
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -622,7 +625,7 @@ void ClientBase::initLogsOutputStream()
|
||||
}
|
||||
}
|
||||
|
||||
logs_out_stream = std::make_unique<InternalTextLogs>(*wb, stdout_is_a_tty);
|
||||
logs_out_stream = std::make_unique<InternalTextLogs>(*wb, color_logs);
|
||||
}
|
||||
}
|
||||
|
||||
@ -2155,6 +2158,7 @@ void ClientBase::init(int argc, char ** argv)
|
||||
|
||||
stdin_is_a_tty = isatty(STDIN_FILENO);
|
||||
stdout_is_a_tty = isatty(STDOUT_FILENO);
|
||||
stderr_is_a_tty = isatty(STDERR_FILENO);
|
||||
terminal_width = getTerminalWidth();
|
||||
|
||||
Arguments common_arguments{""}; /// 0th argument is ignored.
|
||||
|
||||
@ -173,6 +173,7 @@ protected:
|
||||
|
||||
bool stdin_is_a_tty = false; /// stdin is a terminal.
|
||||
bool stdout_is_a_tty = false; /// stdout is a terminal.
|
||||
bool stderr_is_a_tty = false; /// stderr is a terminal.
|
||||
uint64_t terminal_width = 0;
|
||||
|
||||
ServerConnectionPtr connection;
|
||||
|
||||
@ -50,52 +50,58 @@ static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggesti
|
||||
{
|
||||
/// NOTE: Once you will update the completion list,
|
||||
/// do not forget to update 01676_clickhouse_client_autocomplete.sh
|
||||
WriteBufferFromOwnString query;
|
||||
query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM ("
|
||||
"SELECT name FROM system.functions"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.table_engines"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.formats"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.table_functions"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.data_type_families"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.merge_tree_settings"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.settings"
|
||||
" UNION ALL ";
|
||||
String query;
|
||||
|
||||
auto add_subquery = [&](std::string_view select, std::string_view result_column_name)
|
||||
{
|
||||
if (!query.empty())
|
||||
query += " UNION ALL ";
|
||||
query += fmt::format("SELECT * FROM viewIfPermitted({} ELSE null('{} String'))", select, result_column_name);
|
||||
};
|
||||
|
||||
auto add_column = [&](std::string_view column_name, std::string_view table_name, bool distinct, std::optional<Int64> limit)
|
||||
{
|
||||
add_subquery(
|
||||
fmt::format(
|
||||
"SELECT {}{} FROM system.{}{}",
|
||||
(distinct ? "DISTINCT " : ""),
|
||||
column_name,
|
||||
table_name,
|
||||
(limit ? (" LIMIT " + std::to_string(*limit)) : "")),
|
||||
column_name);
|
||||
};
|
||||
|
||||
add_column("name", "functions", false, {});
|
||||
add_column("name", "table_engines", false, {});
|
||||
add_column("name", "formats", false, {});
|
||||
add_column("name", "table_functions", false, {});
|
||||
add_column("name", "data_type_families", false, {});
|
||||
add_column("name", "merge_tree_settings", false, {});
|
||||
add_column("name", "settings", false, {});
|
||||
|
||||
if (!basic_suggestion)
|
||||
{
|
||||
query << "SELECT cluster FROM system.clusters"
|
||||
" UNION ALL "
|
||||
"SELECT macro FROM system.macros"
|
||||
" UNION ALL "
|
||||
"SELECT policy_name FROM system.storage_policies"
|
||||
" UNION ALL ";
|
||||
add_column("cluster", "clusters", false, {});
|
||||
add_column("macro", "macros", false, {});
|
||||
add_column("policy_name", "storage_policies", false, {});
|
||||
}
|
||||
query << "SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate";
|
||||
|
||||
add_subquery("SELECT concat(func.name, comb.name) AS x FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate", "x");
|
||||
|
||||
/// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero.
|
||||
if (suggestion_limit > 0)
|
||||
{
|
||||
String limit_str = toString(suggestion_limit);
|
||||
query << " UNION ALL "
|
||||
"SELECT name FROM system.databases LIMIT " << limit_str
|
||||
<< " UNION ALL "
|
||||
"SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
|
||||
<< " UNION ALL ";
|
||||
|
||||
add_column("name", "databases", false, suggestion_limit);
|
||||
add_column("name", "tables", true, suggestion_limit);
|
||||
if (!basic_suggestion)
|
||||
{
|
||||
query << "SELECT DISTINCT name FROM system.dictionaries LIMIT " << limit_str
|
||||
<< " UNION ALL ";
|
||||
add_column("name", "dictionaries", true, suggestion_limit);
|
||||
}
|
||||
query << "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
|
||||
add_column("name", "columns", true, suggestion_limit);
|
||||
}
|
||||
query << ") WHERE notEmpty(res)";
|
||||
|
||||
return query.str();
|
||||
query = "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + query + ") WHERE notEmpty(res)";
|
||||
return query;
|
||||
}
|
||||
|
||||
template <typename ConnectionType>
|
||||
|
||||
@ -84,7 +84,7 @@ struct DummyJSONParser
|
||||
static Iterator begin() { return {}; }
|
||||
static Iterator end() { return {}; }
|
||||
static size_t size() { return 0; }
|
||||
bool find(const std::string_view &, Element &) const { return false; } /// NOLINT
|
||||
bool find(std::string_view, Element &) const { return false; } /// NOLINT
|
||||
|
||||
#if 0
|
||||
/// Optional: Provides access to an object's element by index.
|
||||
@ -93,7 +93,7 @@ struct DummyJSONParser
|
||||
};
|
||||
|
||||
/// Parses a JSON document, returns the reference to its root element if succeeded.
|
||||
bool parse(const std::string_view &, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
|
||||
bool parse(std::string_view, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
|
||||
|
||||
#if 0
|
||||
/// Optional: Allocates memory to parse JSON documents faster.
|
||||
|
||||
@ -98,7 +98,7 @@ struct RapidJSONParser
|
||||
ALWAYS_INLINE Iterator end() const { return ptr->MemberEnd(); }
|
||||
ALWAYS_INLINE size_t size() const { return ptr->MemberCount(); }
|
||||
|
||||
bool find(const std::string_view & key, Element & result) const
|
||||
bool find(std::string_view key, Element & result) const
|
||||
{
|
||||
auto it = ptr->FindMember(rapidjson::StringRef(key.data(), key.length()));
|
||||
if (it == ptr->MemberEnd())
|
||||
@ -122,7 +122,7 @@ struct RapidJSONParser
|
||||
};
|
||||
|
||||
/// Parses a JSON document, returns the reference to its root element if succeeded.
|
||||
bool parse(const std::string_view & json, Element & result)
|
||||
bool parse(std::string_view json, Element & result)
|
||||
{
|
||||
rapidjson::MemoryStream ms(json.data(), json.size());
|
||||
rapidjson::EncodedInputStream<rapidjson::UTF8<>, rapidjson::MemoryStream> is(ms);
|
||||
|
||||
@ -105,7 +105,7 @@ struct SimdJSONParser
|
||||
ALWAYS_INLINE Iterator end() const { return object.end(); }
|
||||
ALWAYS_INLINE size_t size() const { return object.size(); }
|
||||
|
||||
bool find(const std::string_view & key, Element & result) const
|
||||
bool find(std::string_view key, Element & result) const
|
||||
{
|
||||
auto x = object.at_key(key);
|
||||
if (x.error())
|
||||
@ -131,7 +131,7 @@ struct SimdJSONParser
|
||||
};
|
||||
|
||||
/// Parses a JSON document, returns the reference to its root element if succeeded.
|
||||
bool parse(const std::string_view & json, Element & result)
|
||||
bool parse(std::string_view json, Element & result)
|
||||
{
|
||||
auto document = parser.parse(json.data(), json.size());
|
||||
if (document.error())
|
||||
|
||||
@ -10,7 +10,7 @@ namespace DB
|
||||
{
|
||||
#pragma GCC diagnostic warning "-Wold-style-cast"
|
||||
|
||||
std::string encodeSHA256(const std::string_view & text)
|
||||
std::string encodeSHA256(std::string_view text)
|
||||
{
|
||||
return encodeSHA256(text.data(), text.size());
|
||||
}
|
||||
@ -21,7 +21,7 @@ std::string encodeSHA256(const void * text, size_t size)
|
||||
encodeSHA256(text, size, reinterpret_cast<unsigned char *>(out.data()));
|
||||
return out;
|
||||
}
|
||||
void encodeSHA256(const std::string_view & text, unsigned char * out)
|
||||
void encodeSHA256(std::string_view text, unsigned char * out)
|
||||
{
|
||||
encodeSHA256(text.data(), text.size(), out);
|
||||
}
|
||||
|
||||
@ -10,10 +10,10 @@ namespace DB
|
||||
{
|
||||
|
||||
/// Encodes `text` and returns it.
|
||||
std::string encodeSHA256(const std::string_view & text);
|
||||
std::string encodeSHA256(std::string_view text);
|
||||
std::string encodeSHA256(const void * text, size_t size);
|
||||
/// `out` must be at least 32 bytes long.
|
||||
void encodeSHA256(const std::string_view & text, unsigned char * out);
|
||||
void encodeSHA256(std::string_view text, unsigned char * out);
|
||||
void encodeSHA256(const void * text, size_t size, unsigned char * out);
|
||||
|
||||
/// Returns concatenation of error strings for all errors that OpenSSL has recorded, emptying the error queue.
|
||||
|
||||
@ -4,7 +4,7 @@ namespace DB
|
||||
{
|
||||
namespace
|
||||
{
|
||||
SettingChange * find(SettingsChanges & changes, const std::string_view & name)
|
||||
SettingChange * find(SettingsChanges & changes, std::string_view name)
|
||||
{
|
||||
auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
|
||||
if (it == changes.end())
|
||||
@ -12,7 +12,7 @@ namespace
|
||||
return &*it;
|
||||
}
|
||||
|
||||
const SettingChange * find(const SettingsChanges & changes, const std::string_view & name)
|
||||
const SettingChange * find(const SettingsChanges & changes, std::string_view name)
|
||||
{
|
||||
auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
|
||||
if (it == changes.end())
|
||||
@ -21,7 +21,7 @@ namespace
|
||||
}
|
||||
}
|
||||
|
||||
bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) const
|
||||
bool SettingsChanges::tryGet(std::string_view name, Field & out_value) const
|
||||
{
|
||||
const auto * change = find(*this, name);
|
||||
if (!change)
|
||||
@ -30,7 +30,7 @@ bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) c
|
||||
return true;
|
||||
}
|
||||
|
||||
const Field * SettingsChanges::tryGet(const std::string_view & name) const
|
||||
const Field * SettingsChanges::tryGet(std::string_view name) const
|
||||
{
|
||||
const auto * change = find(*this, name);
|
||||
if (!change)
|
||||
@ -38,7 +38,7 @@ const Field * SettingsChanges::tryGet(const std::string_view & name) const
|
||||
return &change->value;
|
||||
}
|
||||
|
||||
Field * SettingsChanges::tryGet(const std::string_view & name)
|
||||
Field * SettingsChanges::tryGet(std::string_view name)
|
||||
{
|
||||
auto * change = find(*this, name);
|
||||
if (!change)
|
||||
|
||||
@ -14,8 +14,8 @@ struct SettingChange
|
||||
Field value;
|
||||
|
||||
SettingChange() = default;
|
||||
SettingChange(const std::string_view & name_, const Field & value_) : name(name_), value(value_) {}
|
||||
SettingChange(const std::string_view & name_, Field && value_) : name(name_), value(std::move(value_)) {}
|
||||
SettingChange(std::string_view name_, const Field & value_) : name(name_), value(value_) {}
|
||||
SettingChange(std::string_view name_, Field && value_) : name(name_), value(std::move(value_)) {}
|
||||
|
||||
friend bool operator ==(const SettingChange & lhs, const SettingChange & rhs) { return (lhs.name == rhs.name) && (lhs.value == rhs.value); }
|
||||
friend bool operator !=(const SettingChange & lhs, const SettingChange & rhs) { return !(lhs == rhs); }
|
||||
@ -27,9 +27,9 @@ class SettingsChanges : public std::vector<SettingChange>
|
||||
public:
|
||||
using std::vector<SettingChange>::vector;
|
||||
|
||||
bool tryGet(const std::string_view & name, Field & out_value) const;
|
||||
const Field * tryGet(const std::string_view & name) const;
|
||||
Field * tryGet(const std::string_view & name);
|
||||
bool tryGet(std::string_view name, Field & out_value) const;
|
||||
const Field * tryGet(std::string_view name) const;
|
||||
Field * tryGet(std::string_view name);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -147,7 +147,7 @@ inline bool isPunctuationASCII(char c)
|
||||
}
|
||||
|
||||
|
||||
inline bool isValidIdentifier(const std::string_view & str)
|
||||
inline bool isValidIdentifier(std::string_view str)
|
||||
{
|
||||
return !str.empty()
|
||||
&& isValidIdentifierBegin(str[0])
|
||||
|
||||
@ -21,6 +21,7 @@
|
||||
|
||||
#if defined(__clang__)
|
||||
#pragma clang diagnostic ignored "-Wgnu-anonymous-struct"
|
||||
#pragma clang diagnostic ignored "-Wnested-anon-types"
|
||||
#endif
|
||||
|
||||
/// Basic idea is motivated by "iotop" tool.
|
||||
|
||||
@ -162,20 +162,19 @@ public:
|
||||
|
||||
template <typename Function, typename... Args>
|
||||
explicit ThreadFromGlobalPool(Function && func, Args &&... args)
|
||||
: state(std::make_shared<Poco::Event>())
|
||||
, thread_id(std::make_shared<std::thread::id>())
|
||||
: state(std::make_shared<State>())
|
||||
{
|
||||
/// NOTE: If this will throw an exception, the destructor won't be called.
|
||||
/// NOTE:
|
||||
/// - If this will throw an exception, the destructor won't be called
|
||||
/// - this pointer cannot be passed in the lambda, since after detach() it will not be valid
|
||||
GlobalThreadPool::instance().scheduleOrThrow([
|
||||
thread_id = thread_id,
|
||||
state = state,
|
||||
func = std::forward<Function>(func),
|
||||
args = std::make_tuple(std::forward<Args>(args)...)]() mutable /// mutable is needed to destroy capture
|
||||
{
|
||||
auto event = std::move(state);
|
||||
SCOPE_EXIT(event->set());
|
||||
SCOPE_EXIT(state->event.set());
|
||||
|
||||
thread_id = std::make_shared<std::thread::id>(std::this_thread::get_id());
|
||||
state->thread_id = std::this_thread::get_id();
|
||||
|
||||
/// This moves are needed to destroy function and arguments before exit.
|
||||
/// It will guarantee that after ThreadFromGlobalPool::join all captured params are destroyed.
|
||||
@ -196,31 +195,30 @@ public:
|
||||
|
||||
ThreadFromGlobalPool & operator=(ThreadFromGlobalPool && rhs) noexcept
|
||||
{
|
||||
if (joinable())
|
||||
if (initialized())
|
||||
abort();
|
||||
state = std::move(rhs.state);
|
||||
thread_id = std::move(rhs.thread_id);
|
||||
return *this;
|
||||
}
|
||||
|
||||
~ThreadFromGlobalPool()
|
||||
{
|
||||
if (joinable())
|
||||
if (initialized())
|
||||
abort();
|
||||
}
|
||||
|
||||
void join()
|
||||
{
|
||||
if (!joinable())
|
||||
if (!initialized())
|
||||
abort();
|
||||
|
||||
state->wait();
|
||||
state->event.wait();
|
||||
state.reset();
|
||||
}
|
||||
|
||||
void detach()
|
||||
{
|
||||
if (!joinable())
|
||||
if (!initialized())
|
||||
abort();
|
||||
state.reset();
|
||||
}
|
||||
@ -230,15 +228,30 @@ public:
|
||||
if (!state)
|
||||
return false;
|
||||
/// Thread cannot join itself.
|
||||
if (*thread_id == std::this_thread::get_id())
|
||||
if (state->thread_id == std::this_thread::get_id())
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
/// The state used in this object and inside the thread job.
|
||||
std::shared_ptr<Poco::Event> state;
|
||||
std::shared_ptr<std::thread::id> thread_id;
|
||||
struct State
|
||||
{
|
||||
/// Should be atomic() because of possible concurrent access between
|
||||
/// assignment and joinable() check.
|
||||
std::atomic<std::thread::id> thread_id;
|
||||
|
||||
/// The state used in this object and inside the thread job.
|
||||
Poco::Event event;
|
||||
};
|
||||
std::shared_ptr<State> state;
|
||||
|
||||
/// Internally initialized() should be used over joinable(),
|
||||
/// since it is enough to know that the thread is initialized,
|
||||
/// and ignore that fact that thread cannot join itself.
|
||||
bool initialized() const
|
||||
{
|
||||
return static_cast<bool>(state);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/Throttler_fwd.h>
|
||||
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
#include <base/sleep.h>
|
||||
@ -57,7 +59,4 @@ private:
|
||||
std::shared_ptr<Throttler> parent;
|
||||
};
|
||||
|
||||
|
||||
using ThrottlerPtr = std::shared_ptr<Throttler>;
|
||||
|
||||
}
|
||||
|
||||
11
src/Common/Throttler_fwd.h
Normal file
11
src/Common/Throttler_fwd.h
Normal file
@ -0,0 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class Throttler;
|
||||
using ThrottlerPtr = std::shared_ptr<Throttler>;
|
||||
|
||||
}
|
||||
@ -87,7 +87,10 @@ BlockDeviceType getBlockDeviceType([[maybe_unused]] const String & device_id)
|
||||
#if defined(OS_LINUX)
|
||||
try
|
||||
{
|
||||
ReadBufferFromFile in("/sys/dev/block/" + device_id + "/queue/rotational");
|
||||
const auto path{std::filesystem::path("/sys/dev/block/") / device_id / "queue/rotational"};
|
||||
if (!std::filesystem::exists(path))
|
||||
return BlockDeviceType::UNKNOWN;
|
||||
ReadBufferFromFile in(path);
|
||||
int rotational;
|
||||
readText(rotational, in);
|
||||
return rotational ? BlockDeviceType::ROT : BlockDeviceType::NONROT;
|
||||
@ -109,7 +112,8 @@ UInt64 getBlockDeviceReadAheadBytes([[maybe_unused]] const String & device_id)
|
||||
#if defined(OS_LINUX)
|
||||
try
|
||||
{
|
||||
ReadBufferFromFile in("/sys/dev/block/" + device_id + "/queue/read_ahead_kb");
|
||||
const auto path{std::filesystem::path("/sys/dev/block/") / device_id / "queue/read_ahead_kb"};
|
||||
ReadBufferFromFile in(path);
|
||||
int read_ahead_kb;
|
||||
readText(read_ahead_kb, in);
|
||||
return read_ahead_kb * 1024;
|
||||
|
||||
@ -12,8 +12,8 @@ struct Exception : public Poco::Exception
|
||||
{
|
||||
explicit Exception(const std::string & msg, int code = 0) : Poco::Exception(msg, code) {}
|
||||
int errnum() const { return code(); }
|
||||
const char * name() const throw() override { return "mysqlxx::Exception"; }
|
||||
const char * className() const throw() override { return "mysqlxx::Exception"; }
|
||||
const char * name() const noexcept override { return "mysqlxx::Exception"; }
|
||||
const char * className() const noexcept override { return "mysqlxx::Exception"; }
|
||||
};
|
||||
|
||||
|
||||
@ -21,8 +21,8 @@ struct Exception : public Poco::Exception
|
||||
struct ConnectionFailed : public Exception
|
||||
{
|
||||
explicit ConnectionFailed(const std::string & msg, int code = 0) : Exception(msg, code) {}
|
||||
const char * name() const throw() override { return "mysqlxx::ConnectionFailed"; }
|
||||
const char * className() const throw() override { return "mysqlxx::ConnectionFailed"; }
|
||||
const char * name() const noexcept override { return "mysqlxx::ConnectionFailed"; }
|
||||
const char * className() const noexcept override { return "mysqlxx::ConnectionFailed"; }
|
||||
};
|
||||
|
||||
|
||||
@ -30,8 +30,8 @@ struct ConnectionFailed : public Exception
|
||||
struct ConnectionLost : public Exception
|
||||
{
|
||||
explicit ConnectionLost(const std::string & msg, int code = 0) : Exception(msg, code) {}
|
||||
const char * name() const throw() override { return "mysqlxx::ConnectionLost"; }
|
||||
const char * className() const throw() override { return "mysqlxx::ConnectionLost"; }
|
||||
const char * name() const noexcept override { return "mysqlxx::ConnectionLost"; }
|
||||
const char * className() const noexcept override { return "mysqlxx::ConnectionLost"; }
|
||||
};
|
||||
|
||||
|
||||
@ -39,8 +39,8 @@ struct ConnectionLost : public Exception
|
||||
struct BadQuery : public Exception
|
||||
{
|
||||
explicit BadQuery(const std::string & msg, int code = 0) : Exception(msg, code) {}
|
||||
const char * name() const throw() override { return "mysqlxx::BadQuery"; }
|
||||
const char * className() const throw() override { return "mysqlxx::BadQuery"; }
|
||||
const char * name() const noexcept override { return "mysqlxx::BadQuery"; }
|
||||
const char * className() const noexcept override { return "mysqlxx::BadQuery"; }
|
||||
};
|
||||
|
||||
|
||||
@ -48,8 +48,8 @@ struct BadQuery : public Exception
|
||||
struct CannotParseValue : public Exception
|
||||
{
|
||||
explicit CannotParseValue(const std::string & msg, int code = 0) : Exception(msg, code) {}
|
||||
const char * name() const throw() override { return "mysqlxx::CannotParseValue"; }
|
||||
const char * className() const throw() override { return "mysqlxx::CannotParseValue"; }
|
||||
const char * name() const noexcept override { return "mysqlxx::CannotParseValue"; }
|
||||
const char * className() const noexcept override { return "mysqlxx::CannotParseValue"; }
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -1,36 +1,28 @@
|
||||
#pragma once
|
||||
#include <base/scope_guard.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/LockMemoryExceptionInThread.h>
|
||||
|
||||
|
||||
#define NOEXCEPT_SCOPE_IMPL_CONCAT(n, expected) \
|
||||
LockMemoryExceptionInThread lock_memory_tracker##n(VariableContext::Global); \
|
||||
SCOPE_EXIT( \
|
||||
{ \
|
||||
const auto uncaught = std::uncaught_exceptions(); \
|
||||
assert((expected) == uncaught || (expected) + 1 == uncaught); \
|
||||
if ((expected) < uncaught) \
|
||||
{ \
|
||||
tryLogCurrentException("NOEXCEPT_SCOPE"); \
|
||||
abort(); \
|
||||
} \
|
||||
} \
|
||||
)
|
||||
|
||||
#define NOEXCEPT_SCOPE_IMPL(n, expected) NOEXCEPT_SCOPE_IMPL_CONCAT(n, expected)
|
||||
|
||||
#define NOEXCEPT_SCOPE_CONCAT(n) \
|
||||
const auto num_curr_exceptions##n = std::uncaught_exceptions(); \
|
||||
NOEXCEPT_SCOPE_IMPL(n, num_curr_exceptions##n)
|
||||
|
||||
#define NOEXCEPT_SCOPE_FWD(n) NOEXCEPT_SCOPE_CONCAT(n)
|
||||
|
||||
|
||||
/// It can be used in critical places to exit on unexpected exceptions.
|
||||
/// SIGABRT is usually better that broken in-memory state with unpredictable consequences.
|
||||
/// It also temporarily disables exception from memory tracker in current thread.
|
||||
/// Strict version does not take into account nested exception (i.e. it aborts even when we're in catch block).
|
||||
|
||||
#define NOEXCEPT_SCOPE_STRICT NOEXCEPT_SCOPE_IMPL(__LINE__, 0)
|
||||
#define NOEXCEPT_SCOPE NOEXCEPT_SCOPE_FWD(__LINE__)
|
||||
#define NOEXCEPT_SCOPE_IMPL(...) do { \
|
||||
LockMemoryExceptionInThread \
|
||||
noexcept_lock_memory_tracker(VariableContext::Global); \
|
||||
try \
|
||||
{ \
|
||||
__VA_ARGS__; \
|
||||
} \
|
||||
catch (...) \
|
||||
{ \
|
||||
DB::tryLogCurrentException(__PRETTY_FUNCTION__); \
|
||||
std::terminate(); \
|
||||
} \
|
||||
} while (0) /* to allow leading semi-colon */
|
||||
|
||||
#define NOEXCEPT_SCOPE_STRICT(...) \
|
||||
if (std::uncaught_exceptions()) std::terminate(); \
|
||||
NOEXCEPT_SCOPE_IMPL(__VA_ARGS__)
|
||||
|
||||
#define NOEXCEPT_SCOPE(...) NOEXCEPT_SCOPE_IMPL(__VA_ARGS__)
|
||||
|
||||
@ -32,7 +32,7 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob
|
||||
}
|
||||
std::string escaped_with_globs = buf_for_escaping.str();
|
||||
|
||||
static const re2::RE2 enum_or_range(R"({([\d]+\.\.[\d]+|[^{}*,]+,[^{}*]*[^{}*,])})"); /// regexp for {expr1,expr2,expr3} or {M..N}, where M and N - non-negative integers, expr's should be without {}*,
|
||||
static const re2::RE2 enum_or_range(R"({([\d]+\.\.[\d]+|[^{}*,]+,[^{}*]*[^{}*,])})"); /// regexp for {expr1,expr2,expr3} or {M..N}, where M and N - non-negative integers, expr's should be without "{", "}", "*" and ","
|
||||
re2::StringPiece input(escaped_with_globs);
|
||||
re2::StringPiece matched;
|
||||
std::ostringstream oss_for_replacing; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
|
||||
@ -50,16 +50,32 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob
|
||||
char point;
|
||||
ReadBufferFromString buf_range(buffer);
|
||||
buf_range >> range_begin >> point >> point >> range_end;
|
||||
|
||||
size_t range_begin_width = buffer.find('.');
|
||||
size_t range_end_width = buffer.size() - buffer.find_last_of('.') - 1;
|
||||
bool leading_zeros = buffer[0] == '0';
|
||||
size_t num_len = std::to_string(range_end).size();
|
||||
size_t output_width = 0;
|
||||
|
||||
if (range_begin > range_end) //Descending Sequence {20..15} {9..01}
|
||||
{
|
||||
std::swap(range_begin,range_end);
|
||||
leading_zeros = buffer[buffer.find_last_of('.')+1]=='0';
|
||||
std::swap(range_begin_width,range_end_width);
|
||||
}
|
||||
if (range_begin_width == 1 && leading_zeros)
|
||||
output_width = 1; ///Special Case: {0..10} {0..999}
|
||||
else
|
||||
output_width = std::max(range_begin_width, range_end_width);
|
||||
|
||||
if (leading_zeros)
|
||||
oss_for_replacing << std::setfill('0') << std::setw(num_len);
|
||||
oss_for_replacing << std::setfill('0') << std::setw(output_width);
|
||||
oss_for_replacing << range_begin;
|
||||
|
||||
for (size_t i = range_begin + 1; i <= range_end; ++i)
|
||||
{
|
||||
oss_for_replacing << '|';
|
||||
if (leading_zeros)
|
||||
oss_for_replacing << std::setfill('0') << std::setw(num_len);
|
||||
oss_for_replacing << std::setfill('0') << std::setw(output_width);
|
||||
oss_for_replacing << i;
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,14 +8,36 @@ using namespace DB;
|
||||
|
||||
TEST(Common, makeRegexpPatternFromGlobs)
|
||||
{
|
||||
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("?"), "[^/]");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("*"), "[^/]*");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("/?"), "/[^/]");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("/*"), "/[^/]*");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("*_{{a,b,c,d}}/?.csv"), "[^/]*_\\{(a|b|c|d)\\}/[^/]\\.csv");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..09}"), "f(1|2|3|4|5|6|7|8|9)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..9}"), "f(1|2|3|4|5|6|7|8|9)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{0001..0000009}"), "f(1|2|3|4|5|6|7|8|9)");
|
||||
/* Regex Parsing for {..} can have three possible cases
|
||||
1) The left range width == the right range width
|
||||
2) The left range width > the right range width
|
||||
3) The left range width < the right range width
|
||||
*/
|
||||
// Ascending Sequences
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{1..9}"), "f(1|2|3|4|5|6|7|8|9)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{0..10}"), "f(0|1|2|3|4|5|6|7|8|9|10)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{10..20}"), "f(10|11|12|13|14|15|16|17|18|19|20)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{00..10}"), "f(00|01|02|03|04|05|06|07|08|09|10)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{0001..0009}"), "f(0001|0002|0003|0004|0005|0006|0007|0008|0009)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..9}"), "f(01|02|03|04|05|06|07|08|09)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{000..9}"), "f(000|001|002|003|004|005|006|007|008|009)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{95..103}"), "f(95|96|97|98|99|100|101|102|103)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{99..109}"), "f(99|100|101|102|103|104|105|106|107|108|109)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{001..0009}"), "f(0001|0002|0003|0004|0005|0006|0007|0008|0009)");
|
||||
// Descending Sequences
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{20..15}"), "f(15|16|17|18|19|20)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{200..199}"), "f(199|200)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{0009..0001}"), "f(0001|0002|0003|0004|0005|0006|0007|0008|0009)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{100..90}"), "f(90|91|92|93|94|95|96|97|98|99|100)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{103..95}"), "f(95|96|97|98|99|100|101|102|103)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{9..01}"), "f(01|02|03|04|05|06|07|08|09)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{9..000}"), "f(000|001|002|003|004|005|006|007|008|009)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{1..2}{1..2}"), "f(1|2)(1|2)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{1..1}{1..1}"), "f(1)(1)");
|
||||
EXPECT_EQ(makeRegexpPatternFromGlobs("f{0..0}{0..0}"), "f(0)(0)");
|
||||
|
||||
@ -131,7 +131,7 @@ std::string lastErrorString()
|
||||
/// This function get key and nonce and encrypt text with their help.
|
||||
/// If something went wrong (can't init context or can't encrypt data) it throws exception.
|
||||
/// It returns length of encrypted text.
|
||||
size_t encrypt(const std::string_view & plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce)
|
||||
size_t encrypt(std::string_view plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce)
|
||||
{
|
||||
/// Init context for encryption, using key.
|
||||
EVP_AEAD_CTX encrypt_ctx;
|
||||
@ -160,7 +160,7 @@ size_t encrypt(const std::string_view & plaintext, char * ciphertext_and_tag, En
|
||||
/// This function get key and nonce and encrypt text with their help.
|
||||
/// If something went wrong (can't init context or can't encrypt data) it throws exception.
|
||||
/// It returns length of encrypted text.
|
||||
size_t decrypt(const std::string_view & ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce)
|
||||
size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce)
|
||||
{
|
||||
/// Init context for decryption with given key.
|
||||
EVP_AEAD_CTX decrypt_ctx;
|
||||
|
||||
@ -11,7 +11,7 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_SETTING;
|
||||
}
|
||||
|
||||
void BaseSettingsHelpers::writeString(const std::string_view & str, WriteBuffer & out)
|
||||
void BaseSettingsHelpers::writeString(std::string_view str, WriteBuffer & out)
|
||||
{
|
||||
writeStringBinary(str, out);
|
||||
}
|
||||
@ -39,13 +39,13 @@ BaseSettingsHelpers::Flags BaseSettingsHelpers::readFlags(ReadBuffer & in)
|
||||
}
|
||||
|
||||
|
||||
void BaseSettingsHelpers::throwSettingNotFound(const std::string_view & name)
|
||||
void BaseSettingsHelpers::throwSettingNotFound(std::string_view name)
|
||||
{
|
||||
throw Exception("Unknown setting " + String{name}, ErrorCodes::UNKNOWN_SETTING);
|
||||
}
|
||||
|
||||
|
||||
void BaseSettingsHelpers::warningSettingNotFound(const std::string_view & name)
|
||||
void BaseSettingsHelpers::warningSettingNotFound(std::string_view name)
|
||||
{
|
||||
static auto * log = &Poco::Logger::get("Settings");
|
||||
LOG_WARNING(log, "Unknown setting {}, skipping", name);
|
||||
|
||||
@ -45,16 +45,16 @@ class BaseSettings : public TTraits::Data
|
||||
public:
|
||||
using Traits = TTraits;
|
||||
|
||||
void set(const std::string_view & name, const Field & value);
|
||||
Field get(const std::string_view & name) const;
|
||||
void set(std::string_view name, const Field & value);
|
||||
Field get(std::string_view name) const;
|
||||
|
||||
void setString(const std::string_view & name, const String & value);
|
||||
String getString(const std::string_view & name) const;
|
||||
void setString(std::string_view name, const String & value);
|
||||
String getString(std::string_view name) const;
|
||||
|
||||
bool tryGet(const std::string_view & name, Field & value) const;
|
||||
bool tryGetString(const std::string_view & name, String & value) const;
|
||||
bool tryGet(std::string_view name, Field & value) const;
|
||||
bool tryGetString(std::string_view name, String & value) const;
|
||||
|
||||
bool isChanged(const std::string_view & name) const;
|
||||
bool isChanged(std::string_view name) const;
|
||||
SettingsChanges changes() const;
|
||||
void applyChange(const SettingChange & change);
|
||||
void applyChanges(const SettingsChanges & changes);
|
||||
@ -63,22 +63,22 @@ public:
|
||||
/// Resets all the settings to their default values.
|
||||
void resetToDefault();
|
||||
|
||||
bool has(const std::string_view & name) const { return hasBuiltin(name) || hasCustom(name); }
|
||||
static bool hasBuiltin(const std::string_view & name);
|
||||
bool hasCustom(const std::string_view & name) const;
|
||||
bool has(std::string_view name) const { return hasBuiltin(name) || hasCustom(name); }
|
||||
static bool hasBuiltin(std::string_view name);
|
||||
bool hasCustom(std::string_view name) const;
|
||||
|
||||
const char * getTypeName(const std::string_view & name) const;
|
||||
const char * getDescription(const std::string_view & name) const;
|
||||
const char * getTypeName(std::string_view name) const;
|
||||
const char * getDescription(std::string_view name) const;
|
||||
|
||||
/// Checks if it's possible to assign a field to a specified value and throws an exception if not.
|
||||
/// This function doesn't change the fields, it performs check only.
|
||||
static void checkCanSet(const std::string_view & name, const Field & value);
|
||||
static void checkCanSetString(const std::string_view & name, const String & str);
|
||||
static void checkCanSet(std::string_view name, const Field & value);
|
||||
static void checkCanSetString(std::string_view name, const String & str);
|
||||
|
||||
/// Conversions without changing the fields.
|
||||
static Field castValueUtil(const std::string_view & name, const Field & value);
|
||||
static String valueToStringUtil(const std::string_view & name, const Field & value);
|
||||
static Field stringToValueUtil(const std::string_view & name, const String & str);
|
||||
static Field castValueUtil(std::string_view name, const Field & value);
|
||||
static String valueToStringUtil(std::string_view name, const Field & value);
|
||||
static Field stringToValueUtil(std::string_view name, const String & str);
|
||||
|
||||
void write(WriteBuffer & out, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT) const;
|
||||
void read(ReadBuffer & in, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT);
|
||||
@ -164,19 +164,19 @@ public:
|
||||
Iterator end() const { return allChanged().end(); }
|
||||
|
||||
private:
|
||||
SettingFieldCustom & getCustomSetting(const std::string_view & name);
|
||||
const SettingFieldCustom & getCustomSetting(const std::string_view & name) const;
|
||||
const SettingFieldCustom * tryGetCustomSetting(const std::string_view & name) const;
|
||||
SettingFieldCustom & getCustomSetting(std::string_view name);
|
||||
const SettingFieldCustom & getCustomSetting(std::string_view name) const;
|
||||
const SettingFieldCustom * tryGetCustomSetting(std::string_view name) const;
|
||||
|
||||
std::conditional_t<Traits::allow_custom_settings, CustomSettingMap, boost::blank> custom_settings_map;
|
||||
};
|
||||
|
||||
struct BaseSettingsHelpers
|
||||
{
|
||||
[[noreturn]] static void throwSettingNotFound(const std::string_view & name);
|
||||
static void warningSettingNotFound(const std::string_view & name);
|
||||
[[noreturn]] static void throwSettingNotFound(std::string_view name);
|
||||
static void warningSettingNotFound(std::string_view name);
|
||||
|
||||
static void writeString(const std::string_view & str, WriteBuffer & out);
|
||||
static void writeString(std::string_view str, WriteBuffer & out);
|
||||
static String readString(ReadBuffer & in);
|
||||
|
||||
enum Flags : UInt64
|
||||
@ -190,7 +190,7 @@ struct BaseSettingsHelpers
|
||||
};
|
||||
|
||||
template <typename TTraits>
|
||||
void BaseSettings<TTraits>::set(const std::string_view & name, const Field & value)
|
||||
void BaseSettings<TTraits>::set(std::string_view name, const Field & value)
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -200,7 +200,7 @@ void BaseSettings<TTraits>::set(const std::string_view & name, const Field & val
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
Field BaseSettings<TTraits>::get(const std::string_view & name) const
|
||||
Field BaseSettings<TTraits>::get(std::string_view name) const
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -210,7 +210,7 @@ Field BaseSettings<TTraits>::get(const std::string_view & name) const
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
void BaseSettings<TTraits>::setString(const std::string_view & name, const String & value)
|
||||
void BaseSettings<TTraits>::setString(std::string_view name, const String & value)
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -220,7 +220,7 @@ void BaseSettings<TTraits>::setString(const std::string_view & name, const Strin
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
String BaseSettings<TTraits>::getString(const std::string_view & name) const
|
||||
String BaseSettings<TTraits>::getString(std::string_view name) const
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -230,7 +230,7 @@ String BaseSettings<TTraits>::getString(const std::string_view & name) const
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
bool BaseSettings<TTraits>::tryGet(const std::string_view & name, Field & value) const
|
||||
bool BaseSettings<TTraits>::tryGet(std::string_view name, Field & value) const
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -247,7 +247,7 @@ bool BaseSettings<TTraits>::tryGet(const std::string_view & name, Field & value)
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
bool BaseSettings<TTraits>::tryGetString(const std::string_view & name, String & value) const
|
||||
bool BaseSettings<TTraits>::tryGetString(std::string_view name, String & value) const
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -264,7 +264,7 @@ bool BaseSettings<TTraits>::tryGetString(const std::string_view & name, String &
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
bool BaseSettings<TTraits>::isChanged(const std::string_view & name) const
|
||||
bool BaseSettings<TTraits>::isChanged(std::string_view name) const
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -316,20 +316,20 @@ void BaseSettings<TTraits>::resetToDefault()
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
bool BaseSettings<TTraits>::hasBuiltin(const std::string_view & name)
|
||||
bool BaseSettings<TTraits>::hasBuiltin(std::string_view name)
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
return (accessor.find(name) != static_cast<size_t>(-1));
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
bool BaseSettings<TTraits>::hasCustom(const std::string_view & name) const
|
||||
bool BaseSettings<TTraits>::hasCustom(std::string_view name) const
|
||||
{
|
||||
return tryGetCustomSetting(name);
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
const char * BaseSettings<TTraits>::getTypeName(const std::string_view & name) const
|
||||
const char * BaseSettings<TTraits>::getTypeName(std::string_view name) const
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -341,7 +341,7 @@ const char * BaseSettings<TTraits>::getTypeName(const std::string_view & name) c
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
const char * BaseSettings<TTraits>::getDescription(const std::string_view & name) const
|
||||
const char * BaseSettings<TTraits>::getDescription(std::string_view name) const
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -353,19 +353,19 @@ const char * BaseSettings<TTraits>::getDescription(const std::string_view & name
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
void BaseSettings<TTraits>::checkCanSet(const std::string_view & name, const Field & value)
|
||||
void BaseSettings<TTraits>::checkCanSet(std::string_view name, const Field & value)
|
||||
{
|
||||
castValueUtil(name, value);
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
void BaseSettings<TTraits>::checkCanSetString(const std::string_view & name, const String & str)
|
||||
void BaseSettings<TTraits>::checkCanSetString(std::string_view name, const String & str)
|
||||
{
|
||||
stringToValueUtil(name, str);
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
Field BaseSettings<TTraits>::castValueUtil(const std::string_view & name, const Field & value)
|
||||
Field BaseSettings<TTraits>::castValueUtil(std::string_view name, const Field & value)
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -377,7 +377,7 @@ Field BaseSettings<TTraits>::castValueUtil(const std::string_view & name, const
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
String BaseSettings<TTraits>::valueToStringUtil(const std::string_view & name, const Field & value)
|
||||
String BaseSettings<TTraits>::valueToStringUtil(std::string_view name, const Field & value)
|
||||
{
|
||||
const auto & accessor = Traits::Accessor::instance();
|
||||
if (size_t index = accessor.find(name); index != static_cast<size_t>(-1))
|
||||
@ -389,7 +389,7 @@ String BaseSettings<TTraits>::valueToStringUtil(const std::string_view & name, c
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
Field BaseSettings<TTraits>::stringToValueUtil(const std::string_view & name, const String & str)
|
||||
Field BaseSettings<TTraits>::stringToValueUtil(std::string_view name, const String & str)
|
||||
{
|
||||
try
|
||||
{
|
||||
@ -521,7 +521,7 @@ bool operator!=(const BaseSettings<TTraits> & left, const BaseSettings<TTraits>
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_view & name)
|
||||
SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(std::string_view name)
|
||||
{
|
||||
if constexpr (Traits::allow_custom_settings)
|
||||
{
|
||||
@ -537,7 +537,7 @@ SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_v
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::string_view & name) const
|
||||
const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(std::string_view name) const
|
||||
{
|
||||
if constexpr (Traits::allow_custom_settings)
|
||||
{
|
||||
@ -549,7 +549,7 @@ const SettingFieldCustom & BaseSettings<TTraits>::getCustomSetting(const std::st
|
||||
}
|
||||
|
||||
template <typename TTraits>
|
||||
const SettingFieldCustom * BaseSettings<TTraits>::tryGetCustomSetting(const std::string_view & name) const
|
||||
const SettingFieldCustom * BaseSettings<TTraits>::tryGetCustomSetting(std::string_view name) const
|
||||
{
|
||||
if constexpr (Traits::allow_custom_settings)
|
||||
{
|
||||
@ -780,7 +780,7 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
|
||||
public: \
|
||||
static const Accessor & instance(); \
|
||||
size_t size() const { return field_infos.size(); } \
|
||||
size_t find(const std::string_view & name) const; \
|
||||
size_t find(std::string_view name) const; \
|
||||
const String & getName(size_t index) const { return field_infos[index].name; } \
|
||||
const char * getTypeName(size_t index) const { return field_infos[index].type; } \
|
||||
const char * getDescription(size_t index) const { return field_infos[index].description; } \
|
||||
@ -851,7 +851,7 @@ bool BaseSettings<TTraits>::SettingFieldRef::isObsolete() const
|
||||
\
|
||||
SETTINGS_TRAITS_NAME::Accessor::Accessor() {} \
|
||||
\
|
||||
size_t SETTINGS_TRAITS_NAME::Accessor::find(const std::string_view & name) const \
|
||||
size_t SETTINGS_TRAITS_NAME::Accessor::find(std::string_view name) const \
|
||||
{ \
|
||||
auto it = name_to_index_map.find(name); \
|
||||
if (it != name_to_index_map.end()) \
|
||||
|
||||
@ -286,7 +286,7 @@ String Field::dump() const
|
||||
return applyVisitor(FieldVisitorDump(), *this);
|
||||
}
|
||||
|
||||
Field Field::restoreFromDump(const std::string_view & dump_)
|
||||
Field Field::restoreFromDump(std::string_view dump_)
|
||||
{
|
||||
auto show_error = [&dump_]
|
||||
{
|
||||
|
||||
@ -346,7 +346,7 @@ public:
|
||||
}
|
||||
|
||||
/// Create a string inplace.
|
||||
Field(const std::string_view & str) { create(str.data(), str.size()); } /// NOLINT
|
||||
Field(std::string_view str) { create(str.data(), str.size()); } /// NOLINT
|
||||
Field(const String & str) { create(std::string_view{str}); } /// NOLINT
|
||||
Field(String && str) { create(std::move(str)); } /// NOLINT
|
||||
Field(const char * str) { create(std::string_view{str}); } /// NOLINT
|
||||
@ -403,7 +403,7 @@ public:
|
||||
return *this;
|
||||
}
|
||||
|
||||
Field & operator= (const std::string_view & str);
|
||||
Field & operator= (std::string_view str);
|
||||
Field & operator= (const String & str) { return *this = std::string_view{str}; }
|
||||
Field & operator= (String && str);
|
||||
Field & operator= (const char * str) { return *this = std::string_view{str}; }
|
||||
@ -631,7 +631,7 @@ public:
|
||||
}
|
||||
|
||||
String dump() const;
|
||||
static Field restoreFromDump(const std::string_view & dump_);
|
||||
static Field restoreFromDump(std::string_view dump_);
|
||||
|
||||
private:
|
||||
std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(Types::Which),
|
||||
@ -929,7 +929,7 @@ Field::operator=(T && rhs)
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline Field & Field::operator=(const std::string_view & str)
|
||||
inline Field & Field::operator=(std::string_view str)
|
||||
{
|
||||
if (which != Types::String)
|
||||
{
|
||||
|
||||
@ -96,6 +96,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
|
||||
M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \
|
||||
M(UInt64, max_replicated_fetches_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated fetches. Zero means unlimited. Only has meaning at server startup.", 0) \
|
||||
M(UInt64, max_replicated_sends_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited. Only has meaning at server startup.", 0) \
|
||||
M(UInt64, max_remote_read_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for read. Zero means unlimited. Only has meaning at server startup.", 0) \
|
||||
M(UInt64, max_remote_write_network_bandwidth_for_server, 0, "The maximum speed of data exchange over the network in bytes per second for write. Zero means unlimited. Only has meaning at server startup.", 0) \
|
||||
M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \
|
||||
M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \
|
||||
\
|
||||
@ -286,6 +288,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
|
||||
M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \
|
||||
M(String, metrics_perf_events_list, "", "Comma separated list of perf metrics that will be measured throughout queries' execution. Empty means all events. See PerfEventInfo in sources for the available events.", 0) \
|
||||
M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \
|
||||
M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \
|
||||
M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \
|
||||
M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \
|
||||
\
|
||||
@ -391,6 +394,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
|
||||
M(Bool, log_query_views, true, "Log query dependent views into system.query_views_log table. This setting have effect only when 'log_queries' is true.", 0) \
|
||||
M(String, log_comment, "", "Log comment into system.query_log table and server log. It can be set to arbitrary string no longer than max_query_size.", 0) \
|
||||
M(LogsLevel, send_logs_level, LogsLevel::fatal, "Send server text logs with specified minimum level to client. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \
|
||||
M(String, send_logs_source_regexp, "", "Send server text logs with specified regexp to match log source name. Empty means all sources.", 0) \
|
||||
M(Bool, enable_optimize_predicate_expression, true, "If it is set to true, optimize predicates to subqueries.", 0) \
|
||||
M(Bool, enable_optimize_predicate_expression_to_final_subquery, true, "Allow push predicate to final subquery.", 0) \
|
||||
M(Bool, allow_push_predicate_when_subquery_contains_with, true, "Allows push predicate when subquery contains WITH clause", 0) \
|
||||
|
||||
@ -350,7 +350,7 @@ void SettingFieldURI::readBinary(ReadBuffer & in)
|
||||
}
|
||||
|
||||
|
||||
void SettingFieldEnumHelpers::writeBinary(const std::string_view & str, WriteBuffer & out)
|
||||
void SettingFieldEnumHelpers::writeBinary(std::string_view str, WriteBuffer & out)
|
||||
{
|
||||
writeStringBinary(str, out);
|
||||
}
|
||||
|
||||
@ -146,13 +146,13 @@ struct SettingFieldString
|
||||
String value;
|
||||
bool changed = false;
|
||||
|
||||
explicit SettingFieldString(const std::string_view & str = {}) : value(str) {}
|
||||
explicit SettingFieldString(std::string_view str = {}) : value(str) {}
|
||||
explicit SettingFieldString(const String & str) : SettingFieldString(std::string_view{str}) {}
|
||||
explicit SettingFieldString(String && str) : value(std::move(str)) {}
|
||||
explicit SettingFieldString(const char * str) : SettingFieldString(std::string_view{str}) {}
|
||||
explicit SettingFieldString(const Field & f) : SettingFieldString(f.safeGet<const String &>()) {}
|
||||
|
||||
SettingFieldString & operator =(const std::string_view & str) { value = str; changed = true; return *this; }
|
||||
SettingFieldString & operator =(std::string_view str) { value = str; changed = true; return *this; }
|
||||
SettingFieldString & operator =(const String & str) { *this = std::string_view{str}; return *this; }
|
||||
SettingFieldString & operator =(String && str) { value = std::move(str); changed = true; return *this; }
|
||||
SettingFieldString & operator =(const char * str) { *this = std::string_view{str}; return *this; }
|
||||
@ -256,7 +256,7 @@ struct SettingFieldEnum
|
||||
|
||||
struct SettingFieldEnumHelpers
|
||||
{
|
||||
static void writeBinary(const std::string_view & str, WriteBuffer & out);
|
||||
static void writeBinary(std::string_view str, WriteBuffer & out);
|
||||
static String readBinary(ReadBuffer & in);
|
||||
};
|
||||
|
||||
@ -286,7 +286,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
|
||||
{ \
|
||||
using EnumType = ENUM_TYPE; \
|
||||
static const String & toString(EnumType value); \
|
||||
static EnumType fromString(const std::string_view & str); \
|
||||
static EnumType fromString(std::string_view str); \
|
||||
}; \
|
||||
\
|
||||
using SettingField##NEW_NAME = SettingFieldEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;
|
||||
@ -310,7 +310,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
|
||||
ERROR_CODE_FOR_UNEXPECTED_NAME); \
|
||||
} \
|
||||
\
|
||||
typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(const std::string_view & str) \
|
||||
typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(std::string_view str) \
|
||||
{ \
|
||||
static const std::unordered_map<std::string_view, EnumType> map = [] { \
|
||||
std::unordered_map<std::string_view, EnumType> res; \
|
||||
@ -430,7 +430,7 @@ void SettingFieldMultiEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
|
||||
using EnumType = ENUM_TYPE; \
|
||||
static size_t getEnumSize(); \
|
||||
static const String & toString(EnumType value); \
|
||||
static EnumType fromString(const std::string_view & str); \
|
||||
static EnumType fromString(std::string_view str); \
|
||||
}; \
|
||||
\
|
||||
using SettingField##NEW_NAME = SettingFieldMultiEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;
|
||||
|
||||
@ -54,7 +54,7 @@ std::pair<std::string, std::string> splitName(const std::string & name, bool rev
|
||||
return {name.substr(0, idx), name.substr(idx + 1)};
|
||||
}
|
||||
|
||||
std::pair<std::string_view, std::string_view> splitName(const std::string_view & name, bool reverse)
|
||||
std::pair<std::string_view, std::string_view> splitName(std::string_view name, bool reverse)
|
||||
{
|
||||
auto idx = (reverse ? name.find_last_of('.') : name.find_first_of('.'));
|
||||
if (idx == std::string::npos || idx == 0 || idx + 1 == name.size())
|
||||
|
||||
@ -13,7 +13,7 @@ namespace Nested
|
||||
|
||||
/// Splits name of compound identifier by first/last dot (depending on 'reverse' parameter).
|
||||
std::pair<std::string, std::string> splitName(const std::string & name, bool reverse = false);
|
||||
std::pair<std::string_view, std::string_view> splitName(const std::string_view & name, bool reverse = false);
|
||||
std::pair<std::string_view, std::string_view> splitName(std::string_view name, bool reverse = false);
|
||||
|
||||
/// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot.
|
||||
std::string extractTableName(const std::string & nested_name);
|
||||
|
||||
@ -100,7 +100,6 @@ ReadBufferFromAzureBlobStorageGather::createImplementationBufferImpl(const Strin
|
||||
settings,
|
||||
max_single_read_retries,
|
||||
max_single_download_retries,
|
||||
settings.remote_fs_buffer_size,
|
||||
/* use_external_buffer */true,
|
||||
read_until_position);
|
||||
}
|
||||
|
||||
@ -67,8 +67,8 @@ std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObject( /// NOLI
|
||||
auto settings_ptr = settings.get();
|
||||
|
||||
return std::make_unique<ReadBufferFromAzureBlobStorage>(
|
||||
client.get(), object.absolute_path, read_settings, settings_ptr->max_single_read_retries,
|
||||
settings_ptr->max_single_download_retries, read_settings.remote_fs_buffer_size);
|
||||
client.get(), object.absolute_path, patchSettings(read_settings), settings_ptr->max_single_read_retries,
|
||||
settings_ptr->max_single_download_retries);
|
||||
}
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOLINT
|
||||
@ -77,18 +77,19 @@ std::unique_ptr<ReadBufferFromFileBase> AzureObjectStorage::readObjects( /// NOL
|
||||
std::optional<size_t>,
|
||||
std::optional<size_t>) const
|
||||
{
|
||||
ReadSettings disk_read_settings = patchSettings(read_settings);
|
||||
auto settings_ptr = settings.get();
|
||||
auto reader_impl = std::make_unique<ReadBufferFromAzureBlobStorageGather>(
|
||||
client.get(),
|
||||
objects,
|
||||
settings_ptr->max_single_read_retries,
|
||||
settings_ptr->max_single_download_retries,
|
||||
read_settings);
|
||||
disk_read_settings);
|
||||
|
||||
if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
|
||||
if (disk_read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
|
||||
{
|
||||
auto reader = getThreadPoolReader();
|
||||
return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, read_settings, std::move(reader_impl));
|
||||
return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, disk_read_settings, std::move(reader_impl));
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -104,7 +105,7 @@ std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NO
|
||||
std::optional<ObjectAttributes>,
|
||||
FinalizeCallback && finalize_callback,
|
||||
size_t buf_size,
|
||||
const WriteSettings &)
|
||||
const WriteSettings & write_settings)
|
||||
{
|
||||
if (mode != WriteMode::Rewrite)
|
||||
throw Exception("Azure storage doesn't support append", ErrorCodes::UNSUPPORTED_METHOD);
|
||||
@ -113,7 +114,8 @@ std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NO
|
||||
client.get(),
|
||||
object.absolute_path,
|
||||
settings.get()->max_single_part_upload_size,
|
||||
buf_size);
|
||||
buf_size,
|
||||
patchSettings(write_settings));
|
||||
|
||||
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(finalize_callback), object.absolute_path);
|
||||
}
|
||||
@ -207,7 +209,7 @@ void AzureObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguratio
|
||||
{
|
||||
auto new_settings = getAzureBlobStorageSettings(config, config_prefix, context);
|
||||
settings.set(std::move(new_settings));
|
||||
|
||||
applyRemoteThrottlingSettings(context);
|
||||
/// We don't update client
|
||||
}
|
||||
|
||||
|
||||
@ -11,7 +11,6 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int FS_METADATA_ERROR;
|
||||
}
|
||||
|
||||
FakeMetadataStorageFromDisk::FakeMetadataStorageFromDisk(
|
||||
@ -115,85 +114,6 @@ const IMetadataStorage & FakeMetadataStorageFromDiskTransaction::getStorageForNo
|
||||
return metadata_storage;
|
||||
}
|
||||
|
||||
void FakeMetadataStorageFromDiskTransaction::addOperation(MetadataOperationPtr && operation)
|
||||
{
|
||||
if (state != MetadataFromDiskTransactionState::PREPARING)
|
||||
throw Exception(
|
||||
ErrorCodes::FS_METADATA_ERROR,
|
||||
"Cannot add operations to transaction in {} state, it should be in {} state",
|
||||
toString(state), toString(MetadataFromDiskTransactionState::PREPARING));
|
||||
|
||||
operations.emplace_back(std::move(operation));
|
||||
}
|
||||
|
||||
void FakeMetadataStorageFromDiskTransaction::commit()
|
||||
{
|
||||
if (state != MetadataFromDiskTransactionState::PREPARING)
|
||||
throw Exception(
|
||||
ErrorCodes::FS_METADATA_ERROR,
|
||||
"Cannot commit transaction in {} state, it should be in {} state",
|
||||
toString(state), toString(MetadataFromDiskTransactionState::PREPARING));
|
||||
|
||||
{
|
||||
std::unique_lock lock(metadata_storage.metadata_mutex);
|
||||
for (size_t i = 0; i < operations.size(); ++i)
|
||||
{
|
||||
try
|
||||
{
|
||||
operations[i]->execute();
|
||||
}
|
||||
catch (Exception & ex)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
ex.addMessage(fmt::format("While committing metadata operation #{}", i));
|
||||
state = MetadataFromDiskTransactionState::FAILED;
|
||||
rollback(i);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Do it in "best effort" mode
|
||||
for (size_t i = 0; i < operations.size(); ++i)
|
||||
{
|
||||
try
|
||||
{
|
||||
operations[i]->finalize();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__, fmt::format("Failed to finalize operation #{}", i));
|
||||
}
|
||||
}
|
||||
|
||||
state = MetadataFromDiskTransactionState::COMMITTED;
|
||||
}
|
||||
|
||||
void FakeMetadataStorageFromDiskTransaction::rollback(size_t until_pos)
|
||||
{
|
||||
/// Otherwise everything is alright
|
||||
if (state == MetadataFromDiskTransactionState::FAILED)
|
||||
{
|
||||
for (int64_t i = until_pos; i >= 0; --i)
|
||||
{
|
||||
try
|
||||
{
|
||||
operations[i]->undo();
|
||||
}
|
||||
catch (Exception & ex)
|
||||
{
|
||||
state = MetadataFromDiskTransactionState::PARTIALLY_ROLLED_BACK;
|
||||
ex.addMessage(fmt::format("While rolling back operation #{}", i));
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Nothing to do, transaction committed or not even started to commit
|
||||
}
|
||||
}
|
||||
|
||||
void FakeMetadataStorageFromDiskTransaction::writeStringToFile(const std::string & path, const std::string & data)
|
||||
{
|
||||
auto wb = disk->writeFile(path);
|
||||
|
||||
@ -65,12 +65,6 @@ private:
|
||||
const FakeMetadataStorageFromDisk & metadata_storage;
|
||||
|
||||
std::vector<MetadataOperationPtr> operations;
|
||||
MetadataFromDiskTransactionState state{MetadataFromDiskTransactionState::PREPARING};
|
||||
|
||||
void addOperation(MetadataOperationPtr && operation);
|
||||
|
||||
void rollback(size_t until_pos);
|
||||
|
||||
public:
|
||||
FakeMetadataStorageFromDiskTransaction(
|
||||
const FakeMetadataStorageFromDisk & metadata_storage_, DiskPtr disk_)
|
||||
@ -82,7 +76,7 @@ public:
|
||||
|
||||
const IMetadataStorage & getStorageForNonTransactionalReads() const final;
|
||||
|
||||
void commit() final;
|
||||
void commit() final {}
|
||||
|
||||
void writeStringToFile(const std::string & path, const std::string & data) override;
|
||||
|
||||
|
||||
@ -52,7 +52,7 @@ std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObject( /// NOLIN
|
||||
std::optional<size_t>,
|
||||
std::optional<size_t>) const
|
||||
{
|
||||
return std::make_unique<ReadBufferFromHDFS>(object.absolute_path, object.absolute_path, config, read_settings);
|
||||
return std::make_unique<ReadBufferFromHDFS>(object.absolute_path, object.absolute_path, config, patchSettings(read_settings));
|
||||
}
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObjects( /// NOLINT
|
||||
@ -61,7 +61,7 @@ std::unique_ptr<ReadBufferFromFileBase> HDFSObjectStorage::readObjects( /// NOLI
|
||||
std::optional<size_t>,
|
||||
std::optional<size_t>) const
|
||||
{
|
||||
auto hdfs_impl = std::make_unique<ReadBufferFromHDFSGather>(config, objects, read_settings);
|
||||
auto hdfs_impl = std::make_unique<ReadBufferFromHDFSGather>(config, objects, patchSettings(read_settings));
|
||||
auto buf = std::make_unique<ReadIndirectBufferFromRemoteFS>(std::move(hdfs_impl));
|
||||
return std::make_unique<SeekAvoidingReadBuffer>(std::move(buf), settings->min_bytes_for_seek);
|
||||
}
|
||||
@ -72,7 +72,7 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
|
||||
std::optional<ObjectAttributes> attributes,
|
||||
FinalizeCallback && finalize_callback,
|
||||
size_t buf_size,
|
||||
const WriteSettings &)
|
||||
const WriteSettings & write_settings)
|
||||
{
|
||||
if (attributes.has_value())
|
||||
throw Exception(
|
||||
@ -81,7 +81,7 @@ std::unique_ptr<WriteBufferFromFileBase> HDFSObjectStorage::writeObject( /// NOL
|
||||
|
||||
/// Single O_WRONLY in libhdfs adds O_TRUNC
|
||||
auto hdfs_buffer = std::make_unique<WriteBufferFromHDFS>(
|
||||
object.absolute_path, config, settings->replication, buf_size,
|
||||
object.absolute_path, config, settings->replication, patchSettings(write_settings), buf_size,
|
||||
mode == WriteMode::Rewrite ? O_WRONLY : O_WRONLY | O_APPEND);
|
||||
|
||||
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(hdfs_buffer), std::move(finalize_callback), object.absolute_path);
|
||||
@ -155,8 +155,9 @@ void HDFSObjectStorage::copyObject( /// NOLINT
|
||||
}
|
||||
|
||||
|
||||
void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr)
|
||||
void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr context)
|
||||
{
|
||||
applyRemoteThrottlingSettings(context);
|
||||
}
|
||||
|
||||
std::unique_ptr<IObjectStorage> HDFSObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr)
|
||||
|
||||
@ -47,4 +47,27 @@ const std::string & IObjectStorage::getCacheBasePath() const
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getCacheBasePath() is not implemented for object storage");
|
||||
}
|
||||
|
||||
void IObjectStorage::applyRemoteThrottlingSettings(ContextPtr context)
|
||||
{
|
||||
std::unique_lock lock{throttlers_mutex};
|
||||
remote_read_throttler = context->getRemoteReadThrottler();
|
||||
remote_write_throttler = context->getRemoteWriteThrottler();
|
||||
}
|
||||
|
||||
ReadSettings IObjectStorage::patchSettings(const ReadSettings & read_settings) const
|
||||
{
|
||||
std::unique_lock lock{throttlers_mutex};
|
||||
ReadSettings settings{read_settings};
|
||||
settings.remote_throttler = remote_read_throttler;
|
||||
return settings;
|
||||
}
|
||||
|
||||
WriteSettings IObjectStorage::patchSettings(const WriteSettings & write_settings) const
|
||||
{
|
||||
std::unique_lock lock{throttlers_mutex};
|
||||
WriteSettings settings{write_settings};
|
||||
settings.remote_throttler = remote_write_throttler;
|
||||
return settings;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
|
||||
#include <Poco/Timestamp.h>
|
||||
@ -166,6 +167,19 @@ public:
|
||||
virtual bool supportsCache() const { return false; }
|
||||
|
||||
virtual bool isReadOnly() const { return false; }
|
||||
|
||||
protected:
|
||||
/// Should be called from implementation of applyNewSettings()
|
||||
void applyRemoteThrottlingSettings(ContextPtr context);
|
||||
|
||||
/// Should be used by implementation of read* and write* methods
|
||||
ReadSettings patchSettings(const ReadSettings & read_settings) const;
|
||||
WriteSettings patchSettings(const WriteSettings & write_settings) const;
|
||||
|
||||
private:
|
||||
mutable std::mutex throttlers_mutex;
|
||||
ThrottlerPtr remote_read_throttler;
|
||||
ThrottlerPtr remote_write_throttler;
|
||||
};
|
||||
|
||||
using ObjectStoragePtr = std::shared_ptr<IObjectStorage>;
|
||||
|
||||
@ -87,6 +87,14 @@ DiskObjectStorageMetadataPtr MetadataStorageFromDisk::readMetadataUnlocked(const
|
||||
return metadata;
|
||||
}
|
||||
|
||||
DiskObjectStorageMetadataPtr MetadataStorageFromDisk::readMetadataUnlocked(const std::string & path, std::unique_lock<std::shared_mutex> &) const
|
||||
{
|
||||
auto metadata = std::make_unique<DiskObjectStorageMetadata>(disk->getPath(), object_storage_root_path, path);
|
||||
auto str = readFileToString(path);
|
||||
metadata->deserializeFromString(str);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
DiskObjectStorageMetadataPtr MetadataStorageFromDisk::readMetadata(const std::string & path) const
|
||||
{
|
||||
std::shared_lock lock(metadata_mutex);
|
||||
@ -112,13 +120,7 @@ std::unordered_map<String, String> MetadataStorageFromDisk::getSerializedMetadat
|
||||
|
||||
void MetadataStorageFromDiskTransaction::createHardLink(const std::string & path_from, const std::string & path_to)
|
||||
{
|
||||
auto metadata = metadata_storage.readMetadata(path_from);
|
||||
|
||||
metadata->incrementRefCount();
|
||||
|
||||
writeStringToFile(path_from, metadata->serializeToString());
|
||||
|
||||
addOperation(std::make_unique<CreateHardlinkOperation>(path_from, path_to, *metadata_storage.getDisk()));
|
||||
addOperation(std::make_unique<CreateHardlinkOperation>(path_from, path_to, *metadata_storage.disk, metadata_storage));
|
||||
}
|
||||
|
||||
MetadataTransactionPtr MetadataStorageFromDisk::createTransaction() const
|
||||
@ -177,12 +179,12 @@ void MetadataStorageFromDiskTransaction::commit()
|
||||
toString(state), toString(MetadataFromDiskTransactionState::PREPARING));
|
||||
|
||||
{
|
||||
std::lock_guard lock(metadata_storage.metadata_mutex);
|
||||
std::unique_lock lock(metadata_storage.metadata_mutex);
|
||||
for (size_t i = 0; i < operations.size(); ++i)
|
||||
{
|
||||
try
|
||||
{
|
||||
operations[i]->execute();
|
||||
operations[i]->execute(lock);
|
||||
}
|
||||
catch (Exception & ex)
|
||||
{
|
||||
@ -316,29 +318,12 @@ void MetadataStorageFromDiskTransaction::createMetadataFile(const std::string &
|
||||
|
||||
void MetadataStorageFromDiskTransaction::addBlobToMetadata(const std::string & path, const std::string & blob_name, uint64_t size_in_bytes)
|
||||
{
|
||||
DiskObjectStorageMetadataPtr metadata;
|
||||
if (metadata_storage.exists(path))
|
||||
{
|
||||
metadata = metadata_storage.readMetadata(path);
|
||||
metadata->addObject(blob_name, size_in_bytes);
|
||||
writeStringToFile(path, metadata->serializeToString());
|
||||
}
|
||||
else
|
||||
{
|
||||
createMetadataFile(path, blob_name, size_in_bytes);
|
||||
}
|
||||
addOperation(std::make_unique<AddBlobOperation>(path, blob_name, metadata_storage.object_storage_root_path, size_in_bytes, *metadata_storage.disk, metadata_storage));
|
||||
}
|
||||
|
||||
void MetadataStorageFromDiskTransaction::unlinkMetadata(const std::string & path)
|
||||
{
|
||||
auto metadata = metadata_storage.readMetadata(path);
|
||||
uint32_t ref_count = metadata->getRefCount();
|
||||
if (ref_count != 0)
|
||||
{
|
||||
metadata->decrementRefCount();
|
||||
writeStringToFile(path, metadata->serializeToString());
|
||||
}
|
||||
unlinkFile(path);
|
||||
addOperation(std::make_unique<UnlinkMetadataFileOperation>(path, *metadata_storage.disk, metadata_storage));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -55,9 +55,9 @@ public:
|
||||
|
||||
std::string getObjectStorageRootPath() const override { return object_storage_root_path; }
|
||||
|
||||
private:
|
||||
DiskObjectStorageMetadataPtr readMetadata(const std::string & path) const;
|
||||
|
||||
DiskObjectStorageMetadataPtr readMetadataUnlocked(const std::string & path, std::unique_lock<std::shared_mutex> & lock) const;
|
||||
DiskObjectStorageMetadataPtr readMetadataUnlocked(const std::string & path, std::shared_lock<std::shared_mutex> & lock) const;
|
||||
};
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#include <Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.h>
|
||||
#include <Disks/ObjectStorages/MetadataStorageFromDisk.h>
|
||||
#include <Disks/IDisk.h>
|
||||
#include <Common/getRandomASCIIString.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
@ -24,7 +25,7 @@ SetLastModifiedOperation::SetLastModifiedOperation(const std::string & path_, Po
|
||||
{
|
||||
}
|
||||
|
||||
void SetLastModifiedOperation::execute()
|
||||
void SetLastModifiedOperation::execute(std::unique_lock<std::shared_mutex> &)
|
||||
{
|
||||
old_timestamp = disk.getLastModified(path);
|
||||
disk.setLastModified(path, new_timestamp);
|
||||
@ -41,7 +42,7 @@ UnlinkFileOperation::UnlinkFileOperation(const std::string & path_, IDisk & disk
|
||||
{
|
||||
}
|
||||
|
||||
void UnlinkFileOperation::execute()
|
||||
void UnlinkFileOperation::execute(std::unique_lock<std::shared_mutex> &)
|
||||
{
|
||||
auto buf = disk.readFile(path);
|
||||
readStringUntilEOF(prev_data, *buf);
|
||||
@ -61,7 +62,7 @@ CreateDirectoryOperation::CreateDirectoryOperation(const std::string & path_, ID
|
||||
{
|
||||
}
|
||||
|
||||
void CreateDirectoryOperation::execute()
|
||||
void CreateDirectoryOperation::execute(std::unique_lock<std::shared_mutex> &)
|
||||
{
|
||||
disk.createDirectory(path);
|
||||
}
|
||||
@ -77,7 +78,7 @@ CreateDirectoryRecursiveOperation::CreateDirectoryRecursiveOperation(const std::
|
||||
{
|
||||
}
|
||||
|
||||
void CreateDirectoryRecursiveOperation::execute()
|
||||
void CreateDirectoryRecursiveOperation::execute(std::unique_lock<std::shared_mutex> &)
|
||||
{
|
||||
namespace fs = std::filesystem;
|
||||
fs::path p(path);
|
||||
@ -104,7 +105,7 @@ RemoveDirectoryOperation::RemoveDirectoryOperation(const std::string & path_, ID
|
||||
{
|
||||
}
|
||||
|
||||
void RemoveDirectoryOperation::execute()
|
||||
void RemoveDirectoryOperation::execute(std::unique_lock<std::shared_mutex> &)
|
||||
{
|
||||
disk.removeDirectory(path);
|
||||
}
|
||||
@ -121,7 +122,7 @@ RemoveRecursiveOperation::RemoveRecursiveOperation(const std::string & path_, ID
|
||||
{
|
||||
}
|
||||
|
||||
void RemoveRecursiveOperation:: execute()
|
||||
void RemoveRecursiveOperation::execute(std::unique_lock<std::shared_mutex> &)
|
||||
{
|
||||
if (disk.isFile(path))
|
||||
disk.moveFile(path, temp_path);
|
||||
@ -146,20 +147,31 @@ void RemoveRecursiveOperation::finalize()
|
||||
disk.removeRecursive(path);
|
||||
}
|
||||
|
||||
CreateHardlinkOperation::CreateHardlinkOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_)
|
||||
CreateHardlinkOperation::CreateHardlinkOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_, const MetadataStorageFromDisk & metadata_storage_)
|
||||
: path_from(path_from_)
|
||||
, path_to(path_to_)
|
||||
, disk(disk_)
|
||||
, metadata_storage(metadata_storage_)
|
||||
{
|
||||
}
|
||||
|
||||
void CreateHardlinkOperation::execute()
|
||||
void CreateHardlinkOperation::execute(std::unique_lock<std::shared_mutex> & lock)
|
||||
{
|
||||
auto metadata = metadata_storage.readMetadataUnlocked(path_from, lock);
|
||||
|
||||
metadata->incrementRefCount();
|
||||
|
||||
write_operation = std::make_unique<WriteFileOperation>(path_from, disk, metadata->serializeToString());
|
||||
|
||||
write_operation->execute(lock);
|
||||
|
||||
disk.createHardLink(path_from, path_to);
|
||||
}
|
||||
|
||||
void CreateHardlinkOperation::undo()
|
||||
{
|
||||
if (write_operation)
|
||||
write_operation->undo();
|
||||
disk.removeFile(path_to);
|
||||
}
|
||||
|
||||
@ -170,7 +182,7 @@ MoveFileOperation::MoveFileOperation(const std::string & path_from_, const std::
|
||||
{
|
||||
}
|
||||
|
||||
void MoveFileOperation::execute()
|
||||
void MoveFileOperation::execute(std::unique_lock<std::shared_mutex> &)
|
||||
{
|
||||
disk.moveFile(path_from, path_to);
|
||||
}
|
||||
@ -187,7 +199,7 @@ MoveDirectoryOperation::MoveDirectoryOperation(const std::string & path_from_, c
|
||||
{
|
||||
}
|
||||
|
||||
void MoveDirectoryOperation::execute()
|
||||
void MoveDirectoryOperation::execute(std::unique_lock<std::shared_mutex> &)
|
||||
{
|
||||
disk.moveDirectory(path_from, path_to);
|
||||
}
|
||||
@ -197,7 +209,6 @@ void MoveDirectoryOperation::undo()
|
||||
disk.moveDirectory(path_to, path_from);
|
||||
}
|
||||
|
||||
|
||||
ReplaceFileOperation::ReplaceFileOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_)
|
||||
: path_from(path_from_)
|
||||
, path_to(path_to_)
|
||||
@ -206,7 +217,7 @@ ReplaceFileOperation::ReplaceFileOperation(const std::string & path_from_, const
|
||||
{
|
||||
}
|
||||
|
||||
void ReplaceFileOperation::execute()
|
||||
void ReplaceFileOperation::execute(std::unique_lock<std::shared_mutex> &)
|
||||
{
|
||||
if (disk.exists(path_to))
|
||||
disk.moveFile(path_to, temp_path_to);
|
||||
@ -232,7 +243,7 @@ WriteFileOperation::WriteFileOperation(const std::string & path_, IDisk & disk_,
|
||||
{
|
||||
}
|
||||
|
||||
void WriteFileOperation::execute()
|
||||
void WriteFileOperation::execute(std::unique_lock<std::shared_mutex> &)
|
||||
{
|
||||
if (disk.exists(path))
|
||||
{
|
||||
@ -258,4 +269,62 @@ void WriteFileOperation::undo()
|
||||
}
|
||||
}
|
||||
|
||||
void AddBlobOperation::execute(std::unique_lock<std::shared_mutex> & metadata_lock)
|
||||
{
|
||||
DiskObjectStorageMetadataPtr metadata;
|
||||
if (metadata_storage.exists(path))
|
||||
metadata = metadata_storage.readMetadataUnlocked(path, metadata_lock);
|
||||
else
|
||||
metadata = std::make_unique<DiskObjectStorageMetadata>(disk.getPath(), root_path, path);
|
||||
|
||||
metadata->addObject(blob_name, size_in_bytes);
|
||||
|
||||
write_operation = std::make_unique<WriteFileOperation>(path, disk, metadata->serializeToString());
|
||||
|
||||
write_operation->execute(metadata_lock);
|
||||
}
|
||||
|
||||
void AddBlobOperation::undo()
|
||||
{
|
||||
if (write_operation)
|
||||
write_operation->undo();
|
||||
}
|
||||
|
||||
void UnlinkMetadataFileOperation::execute(std::unique_lock<std::shared_mutex> & metadata_lock)
|
||||
{
|
||||
auto metadata = metadata_storage.readMetadataUnlocked(path, metadata_lock);
|
||||
uint32_t ref_count = metadata->getRefCount();
|
||||
if (ref_count != 0)
|
||||
{
|
||||
metadata->decrementRefCount();
|
||||
write_operation = std::make_unique<WriteFileOperation>(path, disk, metadata->serializeToString());
|
||||
write_operation->execute(metadata_lock);
|
||||
}
|
||||
unlink_operation = std::make_unique<UnlinkFileOperation>(path, disk);
|
||||
unlink_operation->execute(metadata_lock);
|
||||
}
|
||||
|
||||
void UnlinkMetadataFileOperation::undo()
|
||||
{
|
||||
if (write_operation)
|
||||
write_operation->undo();
|
||||
|
||||
if (unlink_operation)
|
||||
unlink_operation->undo();
|
||||
}
|
||||
|
||||
void SetReadonlyFileOperation::execute(std::unique_lock<std::shared_mutex> & metadata_lock)
|
||||
{
|
||||
auto metadata = metadata_storage.readMetadataUnlocked(path, metadata_lock);
|
||||
metadata->setReadOnly();
|
||||
write_operation = std::make_unique<WriteFileOperation>(path, disk, metadata->serializeToString());
|
||||
write_operation->execute(metadata_lock);
|
||||
}
|
||||
|
||||
void SetReadonlyFileOperation::undo()
|
||||
{
|
||||
if (write_operation)
|
||||
write_operation->undo();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
class MetadataStorageFromDisk;
|
||||
class IDisk;
|
||||
|
||||
/**
|
||||
@ -12,7 +13,7 @@ class IDisk;
|
||||
|
||||
struct IMetadataOperation
|
||||
{
|
||||
virtual void execute() = 0;
|
||||
virtual void execute(std::unique_lock<std::shared_mutex> & metadata_lock) = 0;
|
||||
virtual void undo() = 0;
|
||||
virtual void finalize() {}
|
||||
virtual ~IMetadataOperation() = default;
|
||||
@ -25,7 +26,7 @@ struct SetLastModifiedOperation final : public IMetadataOperation
|
||||
{
|
||||
SetLastModifiedOperation(const std::string & path_, Poco::Timestamp new_timestamp_, IDisk & disk_);
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
@ -41,7 +42,7 @@ struct UnlinkFileOperation final : public IMetadataOperation
|
||||
{
|
||||
UnlinkFileOperation(const std::string & path_, IDisk & disk_);
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
@ -56,7 +57,7 @@ struct CreateDirectoryOperation final : public IMetadataOperation
|
||||
{
|
||||
CreateDirectoryOperation(const std::string & path_, IDisk & disk_);
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
@ -70,7 +71,7 @@ struct CreateDirectoryRecursiveOperation final : public IMetadataOperation
|
||||
{
|
||||
CreateDirectoryRecursiveOperation(const std::string & path_, IDisk & disk_);
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
@ -85,7 +86,7 @@ struct RemoveDirectoryOperation final : public IMetadataOperation
|
||||
{
|
||||
RemoveDirectoryOperation(const std::string & path_, IDisk & disk_);
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
@ -98,7 +99,7 @@ struct RemoveRecursiveOperation final : public IMetadataOperation
|
||||
{
|
||||
RemoveRecursiveOperation(const std::string & path_, IDisk & disk_);
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
@ -110,12 +111,30 @@ private:
|
||||
std::string temp_path;
|
||||
};
|
||||
|
||||
struct WriteFileOperation final : public IMetadataOperation
|
||||
{
|
||||
WriteFileOperation(const std::string & path_, IDisk & disk_, const std::string & data_);
|
||||
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
private:
|
||||
std::string path;
|
||||
IDisk & disk;
|
||||
std::string data;
|
||||
bool existed = false;
|
||||
std::string prev_data;
|
||||
};
|
||||
|
||||
struct CreateHardlinkOperation final : public IMetadataOperation
|
||||
{
|
||||
CreateHardlinkOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_);
|
||||
CreateHardlinkOperation(
|
||||
const std::string & path_from_,
|
||||
const std::string & path_to_,
|
||||
IDisk & disk_,
|
||||
const MetadataStorageFromDisk & metadata_storage_);
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
@ -123,6 +142,8 @@ private:
|
||||
std::string path_from;
|
||||
std::string path_to;
|
||||
IDisk & disk;
|
||||
std::unique_ptr<WriteFileOperation> write_operation;
|
||||
const MetadataStorageFromDisk & metadata_storage;
|
||||
};
|
||||
|
||||
|
||||
@ -130,7 +151,7 @@ struct MoveFileOperation final : public IMetadataOperation
|
||||
{
|
||||
MoveFileOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_);
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
@ -145,7 +166,7 @@ struct MoveDirectoryOperation final : public IMetadataOperation
|
||||
{
|
||||
MoveDirectoryOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_);
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
@ -160,7 +181,7 @@ struct ReplaceFileOperation final : public IMetadataOperation
|
||||
{
|
||||
ReplaceFileOperation(const std::string & path_from_, const std::string & path_to_, IDisk & disk_);
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
@ -173,20 +194,86 @@ private:
|
||||
std::string temp_path_to;
|
||||
};
|
||||
|
||||
|
||||
struct WriteFileOperation final : public IMetadataOperation
|
||||
struct AddBlobOperation final : public IMetadataOperation
|
||||
{
|
||||
WriteFileOperation(const std::string & path_, IDisk & disk_, const std::string & data_);
|
||||
AddBlobOperation(
|
||||
const std::string & path_,
|
||||
const std::string & blob_name_,
|
||||
const std::string & root_path_,
|
||||
uint64_t size_in_bytes_,
|
||||
IDisk & disk_,
|
||||
const MetadataStorageFromDisk & metadata_storage_)
|
||||
: path(path_)
|
||||
, blob_name(blob_name_)
|
||||
, root_path(root_path_)
|
||||
, size_in_bytes(size_in_bytes_)
|
||||
, disk(disk_)
|
||||
, metadata_storage(metadata_storage_)
|
||||
{}
|
||||
|
||||
void execute() override;
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
private:
|
||||
std::string path;
|
||||
std::string blob_name;
|
||||
std::string root_path;
|
||||
uint64_t size_in_bytes;
|
||||
IDisk & disk;
|
||||
const MetadataStorageFromDisk & metadata_storage;
|
||||
|
||||
std::unique_ptr<WriteFileOperation> write_operation;
|
||||
};
|
||||
|
||||
|
||||
struct UnlinkMetadataFileOperation final : public IMetadataOperation
|
||||
{
|
||||
UnlinkMetadataFileOperation(
|
||||
const std::string & path_,
|
||||
IDisk & disk_,
|
||||
const MetadataStorageFromDisk & metadata_storage_)
|
||||
: path(path_)
|
||||
, disk(disk_)
|
||||
, metadata_storage(metadata_storage_)
|
||||
{
|
||||
}
|
||||
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
private:
|
||||
std::string path;
|
||||
IDisk & disk;
|
||||
std::string data;
|
||||
bool existed = false;
|
||||
std::string prev_data;
|
||||
const MetadataStorageFromDisk & metadata_storage;
|
||||
|
||||
std::unique_ptr<WriteFileOperation> write_operation;
|
||||
std::unique_ptr<UnlinkFileOperation> unlink_operation;
|
||||
};
|
||||
|
||||
struct SetReadonlyFileOperation final : public IMetadataOperation
|
||||
{
|
||||
SetReadonlyFileOperation(
|
||||
const std::string & path_,
|
||||
IDisk & disk_,
|
||||
const MetadataStorageFromDisk & metadata_storage_)
|
||||
: path(path_)
|
||||
, disk(disk_)
|
||||
, metadata_storage(metadata_storage_)
|
||||
{
|
||||
}
|
||||
|
||||
void execute(std::unique_lock<std::shared_mutex> & metadata_lock) override;
|
||||
|
||||
void undo() override;
|
||||
|
||||
private:
|
||||
std::string path;
|
||||
IDisk & disk;
|
||||
const MetadataStorageFromDisk & metadata_storage;
|
||||
|
||||
std::unique_ptr<WriteFileOperation> write_operation;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -118,6 +118,10 @@ std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObjects( /// NOLINT
|
||||
std::optional<size_t>,
|
||||
std::optional<size_t>) const
|
||||
{
|
||||
assert(!objects[0].getPathKeyForCache().empty());
|
||||
|
||||
ReadSettings disk_read_settings = patchSettings(read_settings);
|
||||
|
||||
auto settings_ptr = s3_settings.get();
|
||||
|
||||
auto s3_impl = std::make_unique<ReadBufferFromS3Gather>(
|
||||
@ -131,7 +135,7 @@ std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObjects( /// NOLINT
|
||||
if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
|
||||
{
|
||||
auto reader = getThreadPoolReader();
|
||||
return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, read_settings, std::move(s3_impl));
|
||||
return std::make_unique<AsynchronousReadIndirectBufferFromRemoteFS>(reader, disk_read_settings, std::move(s3_impl));
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -153,10 +157,9 @@ std::unique_ptr<ReadBufferFromFileBase> S3ObjectStorage::readObject( /// NOLINT
|
||||
object.absolute_path,
|
||||
version_id,
|
||||
settings_ptr->s3_settings.max_single_read_retries,
|
||||
read_settings);
|
||||
patchSettings(read_settings));
|
||||
}
|
||||
|
||||
|
||||
std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLINT
|
||||
const StoredObject & object,
|
||||
WriteMode mode, // S3 doesn't support append, only rewrite
|
||||
@ -165,6 +168,8 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
|
||||
size_t buf_size,
|
||||
const WriteSettings & /* write_settings */)
|
||||
{
|
||||
WriteSettings disk_write_settings = IObjectStorage::patchSettings(write_settings);
|
||||
|
||||
if (mode != WriteMode::Rewrite)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "S3 doesn't support append to files");
|
||||
|
||||
@ -176,7 +181,8 @@ std::unique_ptr<WriteBufferFromFileBase> S3ObjectStorage::writeObject( /// NOLIN
|
||||
settings_ptr->s3_settings,
|
||||
attributes,
|
||||
buf_size,
|
||||
threadPoolCallbackRunner(getThreadPoolWriter()));
|
||||
threadPoolCallbackRunner(getThreadPoolWriter()),
|
||||
disk_write_settings);
|
||||
|
||||
return std::make_unique<WriteIndirectBufferFromRemoteFS>(
|
||||
std::move(s3_buffer), std::move(finalize_callback), object.absolute_path);
|
||||
@ -481,6 +487,7 @@ void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &
|
||||
{
|
||||
s3_settings.set(getSettings(config, config_prefix, context));
|
||||
client.set(getClient(config, config_prefix, context));
|
||||
applyRemoteThrottlingSettings(context);
|
||||
}
|
||||
|
||||
std::unique_ptr<IObjectStorage> S3ObjectStorage::cloneObjectStorage(
|
||||
|
||||
@ -134,6 +134,8 @@ public:
|
||||
ContextPtr context) override;
|
||||
|
||||
private:
|
||||
ReadSettings patchSettings(const ReadSettings & read_settings) const;
|
||||
|
||||
void setNewSettings(std::unique_ptr<S3ObjectStorageSettings> && s3_settings_);
|
||||
|
||||
void setNewClient(std::unique_ptr<Aws::S3::S3Client> && client_);
|
||||
|
||||
@ -77,18 +77,18 @@ namespace
|
||||
return convertChar(c1) == convertChar(c2);
|
||||
}
|
||||
|
||||
static bool equals(const std::string_view & s1, const std::string_view & s2)
|
||||
static bool equals(std::string_view s1, std::string_view s2)
|
||||
{
|
||||
return (s1.length() == s2.length())
|
||||
&& std::equal(s1.begin(), s1.end(), s2.begin(), [](char c1, char c2) { return convertChar(c1) == convertChar(c2); });
|
||||
}
|
||||
|
||||
static bool less(const std::string_view & s1, const std::string_view & s2)
|
||||
static bool less(std::string_view s1, std::string_view s2)
|
||||
{
|
||||
return std::lexicographical_compare(s1.begin(), s1.end(), s2.begin(), s2.end(), [](char c1, char c2) { return convertChar(c1) < convertChar(c2); });
|
||||
}
|
||||
|
||||
static bool startsWith(const std::string_view & s1, const std::string_view & s2)
|
||||
static bool startsWith(std::string_view s1, std::string_view s2)
|
||||
{
|
||||
return (s1.length() >= s2.length()) && equals(s1.substr(0, s2.length()), s2);
|
||||
}
|
||||
@ -195,7 +195,7 @@ namespace
|
||||
{
|
||||
protected:
|
||||
ProtobufSerializerSingleValue(
|
||||
const std::string_view & column_name_,
|
||||
std::string_view column_name_,
|
||||
const FieldDescriptor & field_descriptor_,
|
||||
const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
: column_name(column_name_)
|
||||
@ -264,7 +264,7 @@ namespace
|
||||
return reader->readFixed<FieldType>();
|
||||
}
|
||||
|
||||
void writeStr(const std::string_view & str)
|
||||
void writeStr(std::string_view str)
|
||||
{
|
||||
if (!str.empty() || !skip_zero_or_empty)
|
||||
writer->writeString(field_tag, str);
|
||||
@ -274,7 +274,7 @@ namespace
|
||||
void readStrAndAppend(PaddedPODArray<UInt8> & str) { reader->readStringAndAppend(str); }
|
||||
|
||||
template <typename DestType>
|
||||
DestType parseFromStr(const std::string_view & str) const
|
||||
DestType parseFromStr(std::string_view str) const
|
||||
{
|
||||
try
|
||||
{
|
||||
@ -307,7 +307,7 @@ namespace
|
||||
return result;
|
||||
}
|
||||
|
||||
[[noreturn]] void incompatibleColumnType(const std::string_view & column_type) const
|
||||
[[noreturn]] void incompatibleColumnType(std::string_view column_type) const
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::DATA_TYPE_INCOMPATIBLE_WITH_PROTOBUF_FIELD,
|
||||
@ -318,7 +318,7 @@ namespace
|
||||
field_descriptor.type_name());
|
||||
}
|
||||
|
||||
[[noreturn]] void cannotConvertValue(const std::string_view & src_value, const std::string_view & src_type_name, const std::string_view & dest_type_name) const
|
||||
[[noreturn]] void cannotConvertValue(std::string_view src_value, std::string_view src_type_name, std::string_view dest_type_name) const
|
||||
{
|
||||
throw Exception(
|
||||
"Could not convert value '" + String{src_value} + "' from type " + String{src_type_name} + " to type "
|
||||
@ -351,7 +351,7 @@ namespace
|
||||
public:
|
||||
using ColumnType = ColumnVector<NumberType>;
|
||||
|
||||
ProtobufSerializerNumber(const std::string_view & column_name_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
ProtobufSerializerNumber(std::string_view column_name_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
: ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_)
|
||||
{
|
||||
setFunctions();
|
||||
@ -590,7 +590,7 @@ namespace
|
||||
using ColumnType = std::conditional_t<is_fixed_string, ColumnFixedString, ColumnString>;
|
||||
|
||||
ProtobufSerializerString(
|
||||
const std::string_view & column_name_,
|
||||
std::string_view column_name_,
|
||||
const std::shared_ptr<const DataTypeFixedString> & fixed_string_data_type_,
|
||||
const google::protobuf::FieldDescriptor & field_descriptor_,
|
||||
const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
@ -604,7 +604,7 @@ namespace
|
||||
}
|
||||
|
||||
ProtobufSerializerString(
|
||||
const std::string_view & column_name_,
|
||||
std::string_view column_name_,
|
||||
const google::protobuf::FieldDescriptor & field_descriptor_,
|
||||
const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
: ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_)
|
||||
@ -727,7 +727,7 @@ namespace
|
||||
{
|
||||
case FieldTypeId::TYPE_INT32:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeInt(parseFromStr<Int32>(str)); };
|
||||
write_function = [this](std::string_view str) { writeInt(parseFromStr<Int32>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readInt(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); };
|
||||
break;
|
||||
@ -735,7 +735,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_SINT32:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeSInt(parseFromStr<Int32>(str)); };
|
||||
write_function = [this](std::string_view str) { writeSInt(parseFromStr<Int32>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readSInt(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); };
|
||||
break;
|
||||
@ -743,7 +743,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_UINT32:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeUInt(parseFromStr<UInt32>(str)); };
|
||||
write_function = [this](std::string_view str) { writeUInt(parseFromStr<UInt32>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readUInt(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_uint32()); };
|
||||
break;
|
||||
@ -751,7 +751,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_INT64:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeInt(parseFromStr<Int64>(str)); };
|
||||
write_function = [this](std::string_view str) { writeInt(parseFromStr<Int64>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readInt(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); };
|
||||
break;
|
||||
@ -759,7 +759,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_SINT64:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeSInt(parseFromStr<Int64>(str)); };
|
||||
write_function = [this](std::string_view str) { writeSInt(parseFromStr<Int64>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readSInt(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); };
|
||||
break;
|
||||
@ -767,7 +767,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_UINT64:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeUInt(parseFromStr<UInt64>(str)); };
|
||||
write_function = [this](std::string_view str) { writeUInt(parseFromStr<UInt64>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readUInt(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_uint64()); };
|
||||
break;
|
||||
@ -775,7 +775,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_FIXED32:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeFixed<UInt32>(parseFromStr<UInt32>(str)); };
|
||||
write_function = [this](std::string_view str) { writeFixed<UInt32>(parseFromStr<UInt32>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<UInt32>(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_uint32()); };
|
||||
break;
|
||||
@ -783,7 +783,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_SFIXED32:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeFixed<Int32>(parseFromStr<Int32>(str)); };
|
||||
write_function = [this](std::string_view str) { writeFixed<Int32>(parseFromStr<Int32>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Int32>(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); };
|
||||
break;
|
||||
@ -791,7 +791,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_FIXED64:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeFixed<UInt64>(parseFromStr<UInt64>(str)); };
|
||||
write_function = [this](std::string_view str) { writeFixed<UInt64>(parseFromStr<UInt64>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<UInt64>(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_uint64()); };
|
||||
break;
|
||||
@ -799,7 +799,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_SFIXED64:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeFixed<Int64>(parseFromStr<Int64>(str)); };
|
||||
write_function = [this](std::string_view str) { writeFixed<Int64>(parseFromStr<Int64>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Int64>(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); };
|
||||
break;
|
||||
@ -807,7 +807,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_FLOAT:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeFixed<Float32>(parseFromStr<Float32>(str)); };
|
||||
write_function = [this](std::string_view str) { writeFixed<Float32>(parseFromStr<Float32>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Float32>(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_float()); };
|
||||
break;
|
||||
@ -815,7 +815,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_DOUBLE:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeFixed<Float64>(parseFromStr<Float64>(str)); };
|
||||
write_function = [this](std::string_view str) { writeFixed<Float64>(parseFromStr<Float64>(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { toStringAppend(readFixed<Float64>(), str); };
|
||||
default_function = [this]() -> String { return toString(field_descriptor.default_value_double()); };
|
||||
break;
|
||||
@ -823,7 +823,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_BOOL:
|
||||
{
|
||||
write_function = [this](const std::string_view & str)
|
||||
write_function = [this](std::string_view str)
|
||||
{
|
||||
if (str == "true")
|
||||
writeUInt(1);
|
||||
@ -855,7 +855,7 @@ namespace
|
||||
case FieldTypeId::TYPE_STRING:
|
||||
case FieldTypeId::TYPE_BYTES:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeStr(str); };
|
||||
write_function = [this](std::string_view str) { writeStr(str); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { readStrAndAppend(str); };
|
||||
default_function = [this]() -> String { return field_descriptor.default_value_string(); };
|
||||
break;
|
||||
@ -863,7 +863,7 @@ namespace
|
||||
|
||||
case FieldTypeId::TYPE_ENUM:
|
||||
{
|
||||
write_function = [this](const std::string_view & str) { writeInt(stringToProtobufEnumValue(str)); };
|
||||
write_function = [this](std::string_view str) { writeInt(stringToProtobufEnumValue(str)); };
|
||||
read_function = [this](PaddedPODArray<UInt8> & str) { protobufEnumValueToStringAppend(readInt(), str); };
|
||||
default_function = [this]() -> String { return field_descriptor.default_value_enum()->name(); };
|
||||
break;
|
||||
@ -908,7 +908,7 @@ namespace
|
||||
}
|
||||
}
|
||||
|
||||
int stringToProtobufEnumValue(const std::string_view & str) const
|
||||
int stringToProtobufEnumValue(std::string_view str) const
|
||||
{
|
||||
auto it = string_to_protobuf_enum_value_map.find(str);
|
||||
if (it == string_to_protobuf_enum_value_map.end())
|
||||
@ -932,7 +932,7 @@ namespace
|
||||
|
||||
const std::shared_ptr<const DataTypeFixedString> fixed_string_data_type;
|
||||
const size_t n = 0;
|
||||
std::function<void(const std::string_view &)> write_function;
|
||||
std::function<void(std::string_view)> write_function;
|
||||
std::function<void(PaddedPODArray<UInt8> &)> read_function;
|
||||
std::function<String()> default_function;
|
||||
std::unordered_map<std::string_view, int> string_to_protobuf_enum_value_map;
|
||||
@ -953,7 +953,7 @@ namespace
|
||||
using BaseClass = ProtobufSerializerNumber<NumberType>;
|
||||
|
||||
ProtobufSerializerEnum(
|
||||
const std::string_view & column_name_,
|
||||
std::string_view column_name_,
|
||||
const std::shared_ptr<const EnumDataType> & enum_data_type_,
|
||||
const FieldDescriptor & field_descriptor_,
|
||||
const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
@ -1067,7 +1067,7 @@ namespace
|
||||
protobuf_enum_value_to_enum_data_type_value_map.emplace(protobuf_enum_value, enum_data_type_value);
|
||||
};
|
||||
|
||||
auto iless = [](const std::string_view & s1, const std::string_view & s2) { return ColumnNameWithProtobufFieldNameComparator::less(s1, s2); };
|
||||
auto iless = [](std::string_view s1, std::string_view s2) { return ColumnNameWithProtobufFieldNameComparator::less(s1, s2); };
|
||||
boost::container::flat_map<std::string_view, int, decltype(iless)> string_to_protobuf_enum_value_map;
|
||||
typename decltype(string_to_protobuf_enum_value_map)::sequence_type string_to_protobuf_enum_value_seq;
|
||||
for (int i : collections::range(enum_descriptor.value_count()))
|
||||
@ -1133,9 +1133,9 @@ namespace
|
||||
|
||||
Int64 readInt() { return ProtobufSerializerSingleValue::readInt(); }
|
||||
void writeInt(Int64 value) { ProtobufSerializerSingleValue::writeInt(value); }
|
||||
void writeStr(const std::string_view & str) { ProtobufSerializerSingleValue::writeStr(str); }
|
||||
void writeStr(std::string_view str) { ProtobufSerializerSingleValue::writeStr(str); }
|
||||
void readStr(String & str) { ProtobufSerializerSingleValue::readStr(str); }
|
||||
[[noreturn]] void cannotConvertValue(const std::string_view & src_value, const std::string_view & src_type_name, const std::string_view & dest_type_name) const { ProtobufSerializerSingleValue::cannotConvertValue(src_value, src_type_name, dest_type_name); }
|
||||
[[noreturn]] void cannotConvertValue(std::string_view src_value, std::string_view src_type_name, std::string_view dest_type_name) const { ProtobufSerializerSingleValue::cannotConvertValue(src_value, src_type_name, dest_type_name); }
|
||||
|
||||
const std::shared_ptr<const EnumDataType> enum_data_type;
|
||||
std::unordered_map<NumberType, int> enum_data_type_value_to_protobuf_enum_value_map;
|
||||
@ -1152,7 +1152,7 @@ namespace
|
||||
using ColumnType = ColumnDecimal<DecimalType>;
|
||||
|
||||
ProtobufSerializerDecimal(
|
||||
const std::string_view & column_name_,
|
||||
std::string_view column_name_,
|
||||
const DataTypeDecimalBase<DecimalType> & decimal_data_type_,
|
||||
const FieldDescriptor & field_descriptor_,
|
||||
const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
@ -1412,7 +1412,7 @@ namespace
|
||||
{
|
||||
public:
|
||||
ProtobufSerializerDate(
|
||||
const std::string_view & column_name_,
|
||||
std::string_view column_name_,
|
||||
const FieldDescriptor & field_descriptor_,
|
||||
const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
: ProtobufSerializerNumber<UInt16>(column_name_, field_descriptor_, reader_or_writer_)
|
||||
@ -1490,7 +1490,7 @@ namespace
|
||||
{
|
||||
public:
|
||||
ProtobufSerializerDateTime(
|
||||
const std::string_view & column_name_,
|
||||
std::string_view column_name_,
|
||||
const DataTypeDateTime & type,
|
||||
const FieldDescriptor & field_descriptor_,
|
||||
const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
@ -1574,7 +1574,7 @@ namespace
|
||||
{
|
||||
public:
|
||||
ProtobufSerializerUUID(
|
||||
const std::string_view & column_name_,
|
||||
std::string_view column_name_,
|
||||
const google::protobuf::FieldDescriptor & field_descriptor_,
|
||||
const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
: ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_)
|
||||
@ -1654,7 +1654,7 @@ namespace
|
||||
{
|
||||
public:
|
||||
ProtobufSerializerAggregateFunction(
|
||||
const std::string_view & column_name_,
|
||||
std::string_view column_name_,
|
||||
const std::shared_ptr<const DataTypeAggregateFunction> & aggregate_function_data_type_,
|
||||
const google::protobuf::FieldDescriptor & field_descriptor_,
|
||||
const ProtobufReaderOrWriter & reader_or_writer_)
|
||||
@ -2061,7 +2061,7 @@ namespace
|
||||
{
|
||||
public:
|
||||
ProtobufSerializerTupleAsArray(
|
||||
const std::string_view & column_name_,
|
||||
std::string_view column_name_,
|
||||
const std::shared_ptr<const DataTypeTuple> & tuple_data_type_,
|
||||
const FieldDescriptor & field_descriptor_,
|
||||
std::vector<std::unique_ptr<ProtobufSerializer>> element_serializers_)
|
||||
@ -2833,7 +2833,7 @@ namespace
|
||||
return field_names;
|
||||
}
|
||||
|
||||
static bool columnNameEqualsToFieldName(const std::string_view & column_name, const FieldDescriptor & field_descriptor)
|
||||
static bool columnNameEqualsToFieldName(std::string_view column_name, const FieldDescriptor & field_descriptor)
|
||||
{
|
||||
std::string_view suffix;
|
||||
return columnNameStartsWithFieldName(column_name, field_descriptor, suffix) && suffix.empty();
|
||||
@ -2844,7 +2844,7 @@ namespace
|
||||
/// which doesn't match to the field's name.
|
||||
/// The function requires that rest part of the column's name to be started with a dot '.' or underline '_',
|
||||
/// but doesn't include those '.' or '_' characters into `suffix`.
|
||||
static bool columnNameStartsWithFieldName(const std::string_view & column_name, const FieldDescriptor & field_descriptor, std::string_view & suffix)
|
||||
static bool columnNameStartsWithFieldName(std::string_view column_name, const FieldDescriptor & field_descriptor, std::string_view & suffix)
|
||||
{
|
||||
size_t matching_length = 0;
|
||||
const MessageDescriptor & containing_type = *field_descriptor.containing_type();
|
||||
@ -2887,7 +2887,7 @@ namespace
|
||||
/// for that case suffixes are also returned.
|
||||
/// This is only the first filter, buildMessageSerializerImpl() does other checks after calling this function.
|
||||
static bool findFieldsByColumnName(
|
||||
const std::string_view & column_name,
|
||||
std::string_view column_name,
|
||||
const MessageDescriptor & message_descriptor,
|
||||
std::vector<std::pair<const FieldDescriptor *, std::string_view /* suffix */>> & out_field_descriptors_with_suffixes,
|
||||
bool google_wrappers_special_treatment)
|
||||
@ -3030,7 +3030,7 @@ namespace
|
||||
used_column_indices_sorted.reserve(num_columns);
|
||||
size_t sequential_column_index = 0;
|
||||
|
||||
auto add_field_serializer = [&](const std::string_view & column_name_,
|
||||
auto add_field_serializer = [&](std::string_view column_name_,
|
||||
std::vector<size_t> && column_indices_,
|
||||
const FieldDescriptor & field_descriptor_,
|
||||
std::unique_ptr<ProtobufSerializer> field_serializer_)
|
||||
@ -3243,7 +3243,7 @@ namespace
|
||||
/// Builds a serializer for one-to-one match:
|
||||
/// one column is serialized as one field in the protobuf message.
|
||||
std::unique_ptr<ProtobufSerializer> buildFieldSerializer(
|
||||
const std::string_view & column_name,
|
||||
std::string_view column_name,
|
||||
const DataTypePtr & data_type,
|
||||
const FieldDescriptor & field_descriptor,
|
||||
bool allow_repeat,
|
||||
@ -3395,7 +3395,7 @@ namespace
|
||||
}
|
||||
}
|
||||
|
||||
[[noreturn]] static void throwFieldNotRepeated(const FieldDescriptor & field_descriptor, const std::string_view & column_name)
|
||||
[[noreturn]] static void throwFieldNotRepeated(const FieldDescriptor & field_descriptor, std::string_view column_name)
|
||||
{
|
||||
if (!field_descriptor.is_repeated())
|
||||
throw Exception(
|
||||
|
||||
@ -196,7 +196,7 @@ template void ProtobufWriter::writeFixed<UInt64>(int field_number, UInt64 value)
|
||||
template void ProtobufWriter::writeFixed<Float32>(int field_number, Float32 value);
|
||||
template void ProtobufWriter::writeFixed<Float64>(int field_number, Float64 value);
|
||||
|
||||
void ProtobufWriter::writeString(int field_number, const std::string_view & str)
|
||||
void ProtobufWriter::writeString(int field_number, std::string_view str)
|
||||
{
|
||||
size_t length = str.length();
|
||||
size_t old_size = buffer.size();
|
||||
|
||||
@ -30,7 +30,7 @@ public:
|
||||
void writeSInt(int field_number, Int64 value);
|
||||
template <typename T>
|
||||
void writeFixed(int field_number, T value);
|
||||
void writeString(int field_number, const std::string_view & str);
|
||||
void writeString(int field_number, std::string_view str);
|
||||
|
||||
void startRepeatedPack();
|
||||
void endRepeatedPack(int field_number, bool skip_if_empty);
|
||||
|
||||
@ -304,7 +304,7 @@ private:
|
||||
|
||||
/// Performs moves of types MoveType::Key and MoveType::ConstKey.
|
||||
template <typename JSONParser>
|
||||
static bool moveToElementByKey(typename JSONParser::Element & element, const std::string_view & key)
|
||||
static bool moveToElementByKey(typename JSONParser::Element & element, std::string_view key)
|
||||
{
|
||||
if (!element.isObject())
|
||||
return false;
|
||||
@ -504,7 +504,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view)
|
||||
{
|
||||
ColumnVector<UInt8> & col_vec = assert_cast<ColumnVector<UInt8> &>(dest);
|
||||
col_vec.insertValue(1);
|
||||
@ -532,7 +532,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName &) { return 0; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view)
|
||||
{
|
||||
/// This function is called only if JSON is valid.
|
||||
/// If JSON isn't valid then `FunctionJSON::Executor::run()` adds default value (=zero) to `dest` without calling this function.
|
||||
@ -556,7 +556,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
size_t size;
|
||||
if (element.isArray())
|
||||
@ -586,7 +586,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view & last_key)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view last_key)
|
||||
{
|
||||
if (last_key.empty())
|
||||
return false;
|
||||
@ -620,7 +620,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
UInt8 type;
|
||||
if (element.isInt64())
|
||||
@ -662,7 +662,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
NumberType value;
|
||||
|
||||
@ -737,7 +737,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
if (!element.isBool())
|
||||
return false;
|
||||
@ -764,7 +764,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
if (element.isNull())
|
||||
return false;
|
||||
@ -1164,7 +1164,7 @@ public:
|
||||
extract_tree = JSONExtractTree<JSONParser>::build(function_name, result_type);
|
||||
}
|
||||
|
||||
bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
return extract_tree->insertResultToColumn(dest, element);
|
||||
}
|
||||
@ -1207,7 +1207,7 @@ public:
|
||||
extract_tree = JSONExtractTree<JSONParser>::build(function_name, value_type);
|
||||
}
|
||||
|
||||
bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
if (!element.isObject())
|
||||
return false;
|
||||
@ -1251,7 +1251,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
ColumnString & col_str = assert_cast<ColumnString &>(dest);
|
||||
auto & chars = col_str.getChars();
|
||||
@ -1355,7 +1355,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
if (!element.isArray())
|
||||
return false;
|
||||
@ -1387,7 +1387,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
if (!element.isObject())
|
||||
return false;
|
||||
@ -1423,7 +1423,7 @@ public:
|
||||
|
||||
static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
|
||||
|
||||
bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &)
|
||||
bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view)
|
||||
{
|
||||
if (!element.isObject())
|
||||
return false;
|
||||
|
||||
@ -562,15 +562,23 @@ public:
|
||||
{
|
||||
const ColumnMap * col_map_left = typeid_cast<const ColumnMap *>(arguments[0].column.get());
|
||||
const auto * col_const_map_left = checkAndGetColumnConst<ColumnMap>(arguments[0].column.get());
|
||||
bool col_const_map_left_flag = false;
|
||||
if (col_const_map_left)
|
||||
{
|
||||
col_const_map_left_flag = true;
|
||||
col_map_left = typeid_cast<const ColumnMap *>(&col_const_map_left->getDataColumn());
|
||||
}
|
||||
if (!col_map_left)
|
||||
return nullptr;
|
||||
|
||||
const ColumnMap * col_map_right = typeid_cast<const ColumnMap *>(arguments[1].column.get());
|
||||
const auto * col_const_map_right = checkAndGetColumnConst<ColumnMap>(arguments[1].column.get());
|
||||
bool col_const_map_right_flag = false;
|
||||
if (col_const_map_right)
|
||||
{
|
||||
col_const_map_right_flag = true;
|
||||
col_map_right = typeid_cast<const ColumnMap *>(&col_const_map_right->getDataColumn());
|
||||
}
|
||||
if (!col_map_right)
|
||||
return nullptr;
|
||||
|
||||
@ -592,13 +600,18 @@ public:
|
||||
MutableColumnPtr offsets = DataTypeNumber<IColumn::Offset>().createColumn();
|
||||
|
||||
IColumn::Offset current_offset = 0;
|
||||
for (size_t idx = 0; idx < input_rows_count; ++idx)
|
||||
for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx)
|
||||
{
|
||||
for (size_t i = offsets_left[idx - 1]; i < offsets_left[idx]; ++i)
|
||||
size_t left_it_begin = col_const_map_left_flag ? 0 : offsets_left[row_idx - 1];
|
||||
size_t left_it_end = col_const_map_left_flag ? offsets_left.size() : offsets_left[row_idx];
|
||||
size_t right_it_begin = col_const_map_right_flag ? 0 : offsets_right[row_idx - 1];
|
||||
size_t right_it_end = col_const_map_right_flag ? offsets_right.size() : offsets_right[row_idx];
|
||||
|
||||
for (size_t i = left_it_begin; i < left_it_end; ++i)
|
||||
{
|
||||
bool matched = false;
|
||||
auto key = keys_data_left.getDataAt(i);
|
||||
for (size_t j = offsets_right[idx - 1]; j < offsets_right[idx]; ++j)
|
||||
for (size_t j = right_it_begin; j < right_it_end; ++j)
|
||||
{
|
||||
if (keys_data_right.getDataAt(j).toString() == key.toString())
|
||||
{
|
||||
@ -613,12 +626,14 @@ public:
|
||||
++current_offset;
|
||||
}
|
||||
}
|
||||
for (size_t j = offsets_right[idx - 1]; j < offsets_right[idx]; ++j)
|
||||
|
||||
for (size_t j = right_it_begin; j < right_it_end; ++j)
|
||||
{
|
||||
keys_data->insertFrom(keys_data_right, j);
|
||||
values_data->insertFrom(values_data_right, j);
|
||||
++current_offset;
|
||||
}
|
||||
|
||||
offsets->insert(current_offset);
|
||||
}
|
||||
|
||||
|
||||
@ -9,6 +9,7 @@ void registerFunctionNotLike(FunctionFactory &);
|
||||
void registerFunctionNotILike(FunctionFactory &);
|
||||
void registerFunctionMatch(FunctionFactory &);
|
||||
void registerFunctionExtract(FunctionFactory &);
|
||||
void registerFunctionTranslate(FunctionFactory &);
|
||||
void registerFunctionReplaceOne(FunctionFactory &);
|
||||
void registerFunctionReplaceAll(FunctionFactory &);
|
||||
void registerFunctionReplaceRegexpOne(FunctionFactory &);
|
||||
@ -31,6 +32,7 @@ void registerFunctionsStringRegexp(FunctionFactory & factory)
|
||||
registerFunctionNotILike(factory);
|
||||
registerFunctionMatch(factory);
|
||||
registerFunctionExtract(factory);
|
||||
registerFunctionTranslate(factory);
|
||||
registerFunctionReplaceOne(factory);
|
||||
registerFunctionReplaceAll(factory);
|
||||
registerFunctionReplaceRegexpOne(factory);
|
||||
|
||||
364
src/Functions/translate.cpp
Normal file
364
src/Functions/translate.cpp
Normal file
@ -0,0 +1,364 @@
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/UTF8Helpers.h>
|
||||
#include <Common/HashTable/HashMap.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
struct TranslateImpl
|
||||
{
|
||||
using Map = std::array<UInt8, 128>;
|
||||
|
||||
static void fillMapWithValues(
|
||||
Map & map,
|
||||
const std::string & map_from,
|
||||
const std::string & map_to)
|
||||
{
|
||||
if (map_from.size() != map_to.size())
|
||||
throw Exception("Second and trird arguments must be the same length", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
std::iota(map.begin(), map.end(), 0);
|
||||
|
||||
for (size_t i = 0; i < map_from.size(); ++i)
|
||||
{
|
||||
if (!isASCII(map_from[i]) || !isASCII(map_to[i]))
|
||||
throw Exception("Second and trird arguments must be ASCII strings", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
map[map_from[i]] = map_to[i];
|
||||
}
|
||||
}
|
||||
|
||||
static void vector(
|
||||
const ColumnString::Chars & data,
|
||||
const ColumnString::Offsets & offsets,
|
||||
const std::string & map_from,
|
||||
const std::string & map_to,
|
||||
ColumnString::Chars & res_data,
|
||||
ColumnString::Offsets & res_offsets)
|
||||
{
|
||||
Map map;
|
||||
fillMapWithValues(map, map_from, map_to);
|
||||
|
||||
res_data.resize(data.size());
|
||||
res_offsets.assign(offsets);
|
||||
|
||||
UInt8 * dst = res_data.data();
|
||||
|
||||
for (UInt64 i = 0; i < offsets.size(); ++i)
|
||||
{
|
||||
const UInt8 * src = data.data() + offsets[i - 1];
|
||||
const UInt8 * src_end = data.data() + offsets[i] - 1;
|
||||
|
||||
while (src < src_end)
|
||||
{
|
||||
if (*src <= ascii_upper_bound)
|
||||
*dst = map[*src];
|
||||
else
|
||||
*dst = *src;
|
||||
|
||||
++src;
|
||||
++dst;
|
||||
}
|
||||
|
||||
/// Technically '\0' can be mapped into other character,
|
||||
/// so we need to process '\0' delimiter separately
|
||||
*dst++ = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static void vectorFixed(
|
||||
const ColumnString::Chars & data,
|
||||
size_t /*n*/,
|
||||
const std::string & map_from,
|
||||
const std::string & map_to,
|
||||
ColumnString::Chars & res_data)
|
||||
{
|
||||
std::array<UInt8, 128> map;
|
||||
fillMapWithValues(map, map_from, map_to);
|
||||
|
||||
res_data.resize(data.size());
|
||||
|
||||
const UInt8 * src = data.data();
|
||||
const UInt8 * src_end = data.data() + data.size();
|
||||
UInt8 * dst = res_data.data();
|
||||
|
||||
while (src < src_end)
|
||||
{
|
||||
if (*src <= ascii_upper_bound)
|
||||
*dst = map[*src];
|
||||
else
|
||||
*dst = *src;
|
||||
|
||||
++src;
|
||||
++dst;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
static constexpr auto ascii_upper_bound = '\x7f';
|
||||
};
|
||||
|
||||
struct TranslateUTF8Impl
|
||||
{
|
||||
using MapASCII = std::array<UInt32, 128>;
|
||||
using MapUTF8 = HashMap<UInt32, UInt32, HashCRC32<UInt32>>;
|
||||
|
||||
static void fillMapWithValues(
|
||||
MapASCII & map_ascii,
|
||||
MapUTF8 & map,
|
||||
const std::string & map_from,
|
||||
const std::string & map_to)
|
||||
{
|
||||
auto map_from_size = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(map_from.data()), map_from.size());
|
||||
auto map_to_size = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(map_to.data()), map_to.size());
|
||||
|
||||
if (map_from_size != map_to_size)
|
||||
throw Exception("Second and trird arguments must be the same length", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
std::iota(map_ascii.begin(), map_ascii.end(), 0);
|
||||
|
||||
const UInt8 * map_from_ptr = reinterpret_cast<const UInt8 *>(map_from.data());
|
||||
const UInt8 * map_from_end = map_from_ptr + map_from.size();
|
||||
const UInt8 * map_to_ptr = reinterpret_cast<const UInt8 *>(map_to.data());
|
||||
const UInt8 * map_to_end = map_to_ptr + map_to.size();
|
||||
|
||||
while (map_from_ptr < map_from_end && map_to_ptr < map_to_end)
|
||||
{
|
||||
size_t len_from = UTF8::seqLength(*map_from_ptr);
|
||||
size_t len_to = UTF8::seqLength(*map_to_ptr);
|
||||
|
||||
std::optional<UInt32> res_from, res_to;
|
||||
|
||||
if (map_from_ptr + len_from <= map_from_end)
|
||||
res_from = UTF8::convertUTF8ToCodePoint(map_from_ptr, len_from);
|
||||
|
||||
if (map_to_ptr + len_to <= map_to_end)
|
||||
res_to = UTF8::convertUTF8ToCodePoint(map_to_ptr, len_to);
|
||||
|
||||
if (!res_from)
|
||||
throw Exception("Second argument must be a valid UTF-8 string", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if (!res_to)
|
||||
throw Exception("Third argument must be a valid UTF-8 string", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if (*map_from_ptr <= ascii_upper_bound)
|
||||
map_ascii[*map_from_ptr] = *res_to;
|
||||
else
|
||||
map[*res_from] = *res_to;
|
||||
|
||||
map_from_ptr += len_from;
|
||||
map_to_ptr += len_to;
|
||||
}
|
||||
}
|
||||
|
||||
static void vector(
|
||||
const ColumnString::Chars & data,
|
||||
const ColumnString::Offsets & offsets,
|
||||
const std::string & map_from,
|
||||
const std::string & map_to,
|
||||
ColumnString::Chars & res_data,
|
||||
ColumnString::Offsets & res_offsets)
|
||||
{
|
||||
MapASCII map_ascii;
|
||||
MapUTF8 map;
|
||||
fillMapWithValues(map_ascii, map, map_from, map_to);
|
||||
|
||||
res_data.resize(data.size());
|
||||
res_offsets.resize(offsets.size());
|
||||
|
||||
UInt8 * dst = res_data.data();
|
||||
UInt64 data_size = 0;
|
||||
|
||||
for (UInt64 i = 0; i < offsets.size(); ++i)
|
||||
{
|
||||
const UInt8 * src = data.data() + offsets[i - 1];
|
||||
const UInt8 * src_end = data.data() + offsets[i] - 1;
|
||||
|
||||
while (src < src_end)
|
||||
{
|
||||
/// Maximum length of UTF-8 sequence is 4 bytes + 1 zero byte
|
||||
if (data_size + 5 > res_data.size())
|
||||
{
|
||||
res_data.resize(data_size * 2 + 5);
|
||||
dst = res_data.data() + data_size;
|
||||
}
|
||||
|
||||
if (*src <= ascii_upper_bound)
|
||||
{
|
||||
size_t dst_len = UTF8::convertCodePointToUTF8(map_ascii[*src], dst, 4);
|
||||
assert(0 < dst_len && dst_len <= 4);
|
||||
|
||||
src += 1;
|
||||
dst += dst_len;
|
||||
data_size += dst_len;
|
||||
continue;
|
||||
}
|
||||
|
||||
size_t src_len = UTF8::seqLength(*src);
|
||||
assert(0 < src_len && src_len <= 4);
|
||||
|
||||
if (src + src_len <= src_end)
|
||||
{
|
||||
auto src_code_point = UTF8::convertUTF8ToCodePoint(src, src_len);
|
||||
|
||||
if (src_code_point)
|
||||
{
|
||||
auto * it = map.find(*src_code_point);
|
||||
if (it != map.end())
|
||||
{
|
||||
size_t dst_len = UTF8::convertCodePointToUTF8(it->getMapped(), dst, 4);
|
||||
assert(0 < dst_len && dst_len <= 4);
|
||||
|
||||
src += src_len;
|
||||
dst += dst_len;
|
||||
data_size += dst_len;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
src_len = src_end - src;
|
||||
}
|
||||
|
||||
memcpy(dst, src, src_len);
|
||||
dst += src_len;
|
||||
src += src_len;
|
||||
data_size += src_len;
|
||||
}
|
||||
|
||||
/// Technically '\0' can be mapped into other character,
|
||||
/// so we need to process '\0' delimiter separately
|
||||
*dst++ = 0;
|
||||
|
||||
++data_size;
|
||||
res_offsets[i] = data_size;
|
||||
}
|
||||
|
||||
res_data.resize(data_size);
|
||||
}
|
||||
|
||||
[[noreturn]] static void vectorFixed(
|
||||
const ColumnString::Chars & /*data*/,
|
||||
size_t /*n*/,
|
||||
const std::string & /*map_from*/,
|
||||
const std::string & /*map_to*/,
|
||||
ColumnString::Chars & /*res_data*/)
|
||||
{
|
||||
throw Exception("Function translateUTF8 does not support FixedString argument", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
private:
|
||||
static constexpr auto ascii_upper_bound = '\x7f';
|
||||
};
|
||||
|
||||
|
||||
template <typename Impl, typename Name>
|
||||
class FunctionTranslate : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = Name::name;
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionTranslate>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 3; }
|
||||
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
if (!isStringOrFixedString(arguments[0]))
|
||||
throw Exception(
|
||||
"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
if (!isStringOrFixedString(arguments[1]))
|
||||
throw Exception(
|
||||
"Illegal type " + arguments[1]->getName() + " of second argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
if (!isStringOrFixedString(arguments[2]))
|
||||
throw Exception(
|
||||
"Illegal type " + arguments[2]->getName() + " of third argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return std::make_shared<DataTypeString>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
|
||||
{
|
||||
const ColumnPtr column_src = arguments[0].column;
|
||||
const ColumnPtr column_map_from = arguments[1].column;
|
||||
const ColumnPtr column_map_to = arguments[2].column;
|
||||
|
||||
if (!isColumnConst(*column_map_from) || !isColumnConst(*column_map_to))
|
||||
throw Exception("2nd and 3rd arguments of function " + getName() + " must be constants.", ErrorCodes::ILLEGAL_COLUMN);
|
||||
|
||||
const IColumn * c1 = arguments[1].column.get();
|
||||
const IColumn * c2 = arguments[2].column.get();
|
||||
const ColumnConst * c1_const = typeid_cast<const ColumnConst *>(c1);
|
||||
const ColumnConst * c2_const = typeid_cast<const ColumnConst *>(c2);
|
||||
String map_from = c1_const->getValue<String>();
|
||||
String map_to = c2_const->getValue<String>();
|
||||
|
||||
if (const ColumnString * col = checkAndGetColumn<ColumnString>(column_src.get()))
|
||||
{
|
||||
auto col_res = ColumnString::create();
|
||||
Impl::vector(col->getChars(), col->getOffsets(), map_from, map_to, col_res->getChars(), col_res->getOffsets());
|
||||
return col_res;
|
||||
}
|
||||
else if (const ColumnFixedString * col_fixed = checkAndGetColumn<ColumnFixedString>(column_src.get()))
|
||||
{
|
||||
auto col_res = ColumnFixedString::create(col_fixed->getN());
|
||||
Impl::vectorFixed(col_fixed->getChars(), col_fixed->getN(), map_from, map_to, col_res->getChars());
|
||||
return col_res;
|
||||
}
|
||||
else
|
||||
throw Exception(
|
||||
"Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
struct NameTranslate
|
||||
{
|
||||
static constexpr auto name = "translate";
|
||||
};
|
||||
|
||||
struct NameTranslateUTF8
|
||||
{
|
||||
static constexpr auto name = "translateUTF8";
|
||||
};
|
||||
|
||||
using FunctionTranslateASCII = FunctionTranslate<TranslateImpl, NameTranslate>;
|
||||
using FunctionTranslateUTF8 = FunctionTranslate<TranslateUTF8Impl, NameTranslateUTF8>;
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionTranslate(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionTranslateASCII>();
|
||||
factory.registerFunction<FunctionTranslateUTF8>();
|
||||
}
|
||||
|
||||
}
|
||||
@ -5,6 +5,7 @@
|
||||
#include <IO/ReadBufferFromAzureBlobStorage.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/Throttler.h>
|
||||
#include <base/sleep.h>
|
||||
|
||||
|
||||
@ -26,7 +27,6 @@ ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage(
|
||||
const ReadSettings & read_settings_,
|
||||
size_t max_single_read_retries_,
|
||||
size_t max_single_download_retries_,
|
||||
size_t tmp_buffer_size_,
|
||||
bool use_external_buffer_,
|
||||
size_t read_until_position_)
|
||||
: ReadBufferFromFileBase(read_settings_.remote_fs_buffer_size, nullptr, 0)
|
||||
@ -34,7 +34,8 @@ ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage(
|
||||
, path(path_)
|
||||
, max_single_read_retries(max_single_read_retries_)
|
||||
, max_single_download_retries(max_single_download_retries_)
|
||||
, tmp_buffer_size(tmp_buffer_size_)
|
||||
, read_settings(read_settings_)
|
||||
, tmp_buffer_size(read_settings.remote_fs_buffer_size)
|
||||
, use_external_buffer(use_external_buffer_)
|
||||
, read_until_position(read_until_position_)
|
||||
{
|
||||
@ -76,6 +77,8 @@ bool ReadBufferFromAzureBlobStorage::nextImpl()
|
||||
try
|
||||
{
|
||||
bytes_read = data_stream->ReadToCount(reinterpret_cast<uint8_t *>(data_ptr), to_read_bytes);
|
||||
if (read_settings.remote_throttler)
|
||||
read_settings.remote_throttler->add(bytes_read);
|
||||
break;
|
||||
}
|
||||
catch (const Azure::Storage::StorageException & e)
|
||||
|
||||
@ -23,7 +23,6 @@ public:
|
||||
const ReadSettings & read_settings_,
|
||||
size_t max_single_read_retries_,
|
||||
size_t max_single_download_retries_,
|
||||
size_t tmp_buffer_size_,
|
||||
bool use_external_buffer_ = false,
|
||||
size_t read_until_position_ = 0);
|
||||
|
||||
@ -48,6 +47,7 @@ private:
|
||||
const String path;
|
||||
size_t max_single_read_retries;
|
||||
size_t max_single_download_retries;
|
||||
ReadSettings read_settings;
|
||||
std::vector<char> tmp_buffer;
|
||||
size_t tmp_buffer_size;
|
||||
bool use_external_buffer;
|
||||
|
||||
@ -5,12 +5,13 @@
|
||||
|
||||
#include <IO/ReadBufferFromIStream.h>
|
||||
#include <IO/ReadBufferFromS3.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
|
||||
#include <aws/s3/S3Client.h>
|
||||
#include <aws/s3/model/GetObjectRequest.h>
|
||||
#include <aws/s3/model/HeadObjectRequest.h>
|
||||
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/Throttler.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/sleep.h>
|
||||
|
||||
@ -164,6 +165,8 @@ bool ReadBufferFromS3::nextImpl()
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::ReadBufferFromS3Bytes, working_buffer.size());
|
||||
offset += working_buffer.size();
|
||||
if (read_settings.remote_throttler)
|
||||
read_settings.remote_throttler->add(working_buffer.size());
|
||||
|
||||
return true;
|
||||
}
|
||||
@ -299,7 +302,6 @@ std::unique_ptr<ReadBuffer> ReadBufferFromS3::initialize()
|
||||
if (outcome.IsSuccess())
|
||||
{
|
||||
read_result = outcome.GetResultWithOwnership();
|
||||
|
||||
size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size;
|
||||
return std::make_unique<ReadBufferFromIStream>(read_result.GetBody(), buffer_size);
|
||||
}
|
||||
|
||||
@ -836,7 +836,7 @@ template <typename T>
|
||||
inline T parse(const char * data, size_t size);
|
||||
|
||||
template <typename T>
|
||||
inline T parseFromString(const std::string_view & str)
|
||||
inline T parseFromString(std::string_view str)
|
||||
{
|
||||
return parse<T>(str.data(), str.size());
|
||||
}
|
||||
@ -1338,7 +1338,7 @@ inline T parseWithSizeSuffix(const char * data, size_t size)
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline T parseWithSizeSuffix(const std::string_view & s)
|
||||
inline T parseWithSizeSuffix(std::string_view s)
|
||||
{
|
||||
return parseWithSizeSuffix<T>(s.data(), s.size());
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <string>
|
||||
#include <Core/Defines.h>
|
||||
#include <Common/FileCache_fwd.h>
|
||||
#include <Common/Throttler_fwd.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -92,6 +93,9 @@ struct ReadSettings
|
||||
|
||||
FileCachePtr remote_fs_cache;
|
||||
|
||||
/// Bandwidth throttler to use during reading
|
||||
ThrottlerPtr remote_throttler;
|
||||
|
||||
size_t http_max_tries = 1;
|
||||
size_t http_retry_initial_backoff_ms = 100;
|
||||
size_t http_retry_max_backoff_ms = 1600;
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <IO/WriteBufferFromAzureBlobStorage.h>
|
||||
#include <Common/getRandomASCIIString.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/Throttler.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -15,11 +16,13 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage(
|
||||
const String & blob_path_,
|
||||
size_t max_single_part_upload_size_,
|
||||
size_t buf_size_,
|
||||
const WriteSettings & write_settings_,
|
||||
std::optional<std::map<std::string, std::string>> attributes_)
|
||||
: BufferWithOwnMemory<WriteBuffer>(buf_size_, nullptr, 0)
|
||||
, blob_container_client(blob_container_client_)
|
||||
, max_single_part_upload_size(max_single_part_upload_size_)
|
||||
, blob_path(blob_path_)
|
||||
, write_settings(write_settings_)
|
||||
, attributes(attributes_)
|
||||
{
|
||||
}
|
||||
@ -84,6 +87,9 @@ void WriteBufferFromAzureBlobStorage::nextImpl()
|
||||
}
|
||||
|
||||
block_blob_client.CommitBlockList(block_ids);
|
||||
|
||||
if (write_settings.remote_throttler)
|
||||
write_settings.remote_throttler->add(read);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user