diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 66ba8547894..b76ff39f0c8 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -2902,7 +2902,7 @@ jobs: ############################################################################################# #################################### PERFORMANCE TESTS ###################################### ############################################################################################# - PerformanceComparison0: + PerformanceComparisonX86-0: needs: [BuilderDebRelease] runs-on: [self-hosted, stress-tester] steps: @@ -2940,7 +2940,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - PerformanceComparison1: + PerformanceComparisonX86-1: needs: [BuilderDebRelease] runs-on: [self-hosted, stress-tester] steps: @@ -2978,7 +2978,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - PerformanceComparison2: + PerformanceComparisonX86-2: needs: [BuilderDebRelease] runs-on: [self-hosted, stress-tester] steps: @@ -3016,7 +3016,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - PerformanceComparison3: + PerformanceComparisonX86-3: needs: [BuilderDebRelease] runs-on: [self-hosted, stress-tester] steps: @@ -3096,10 +3096,10 @@ jobs: - IntegrationTestsTsan1 - IntegrationTestsTsan2 - IntegrationTestsTsan3 - - PerformanceComparison0 - - PerformanceComparison1 - - PerformanceComparison2 - - PerformanceComparison3 + - PerformanceComparisonX86-0 + - PerformanceComparisonX86-1 + - PerformanceComparisonX86-2 + - PerformanceComparisonX86-3 - CompatibilityCheck - ASTFuzzerTestDebug - ASTFuzzerTestAsan diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 9cd8fd6f49d..05fd337e57c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -3118,7 +3118,7 @@ jobs: ############################################################################################# #################################### PERFORMANCE TESTS ###################################### ############################################################################################# - PerformanceComparison0: + PerformanceComparisonX86-0: needs: [BuilderDebRelease] runs-on: [self-hosted, stress-tester] steps: @@ -3156,7 +3156,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - PerformanceComparison1: + PerformanceComparisonX86-1: needs: [BuilderDebRelease] runs-on: [self-hosted, stress-tester] steps: @@ -3194,7 +3194,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - PerformanceComparison2: + PerformanceComparisonX86-2: needs: [BuilderDebRelease] runs-on: [self-hosted, stress-tester] steps: @@ -3232,7 +3232,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - PerformanceComparison3: + PerformanceComparisonX86-3: needs: [BuilderDebRelease] runs-on: [self-hosted, stress-tester] steps: @@ -3270,7 +3270,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch0: + PerformanceComparisonAarch-0: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64] steps: @@ -3308,7 +3308,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch1: + PerformanceComparisonAarch-1: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64] steps: @@ -3346,7 +3346,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch2: + PerformanceComparisonAarch-2: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64] steps: @@ -3384,7 +3384,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch3: + PerformanceComparisonAarch-3: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64] steps: @@ -3481,14 +3481,14 @@ jobs: - IntegrationTestsTsan1 - IntegrationTestsTsan2 - IntegrationTestsTsan3 - - PerformanceComparison0 - - PerformanceComparison1 - - PerformanceComparison2 - - PerformanceComparison3 - - PerformanceComparisonAarch0 - - PerformanceComparisonAarch1 - - PerformanceComparisonAarch2 - - PerformanceComparisonAarch3 + - PerformanceComparisonX86-0 + - PerformanceComparisonX86-1 + - PerformanceComparisonX86-2 + - PerformanceComparisonX86-3 + - PerformanceComparisonAarch-0 + - PerformanceComparisonAarch-1 + - PerformanceComparisonAarch-2 + - PerformanceComparisonAarch-3 - UnitTestsAsan - UnitTestsTsan - UnitTestsMsan diff --git a/CMakeLists.txt b/CMakeLists.txt index d01d2e2179d..367a88140e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -335,6 +335,22 @@ if (COMPILER_GCC OR COMPILER_CLANG) set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32") endif () +if (ARCH_AMD64) + # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change, + # which makes benchmark results more stable. + set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries") + if (COMPILER_GCC) + # gcc is in assembler, need to add "-Wa," prefix + set(BRANCHES_WITHIN_32B_BOUNDARIES "-Wa,${BRANCHES_WITHIN_32B_BOUNDARIES}") + endif() + + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("${BRANCHES_WITHIN_32B_BOUNDARIES}" HAS_BRANCHES_WITHIN_32B_BOUNDARIES) + if (HAS_BRANCHES_WITHIN_32B_BOUNDARIES) + set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}") + endif() +endif() + if (COMPILER_GCC) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines") endif () diff --git a/base/base/JSON.cpp b/base/base/JSON.cpp index 54074f20f16..a01063ee426 100644 --- a/base/base/JSON.cpp +++ b/base/base/JSON.cpp @@ -12,7 +12,14 @@ #define JSON_MAX_DEPTH 100 +#ifdef __clang__ +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec" +#endif POCO_IMPLEMENT_EXCEPTION(JSONException, Poco::Exception, "JSONException") // NOLINT(cert-err60-cpp, modernize-use-noexcept, hicpp-use-noexcept) +#ifdef __clang__ +# pragma clang diagnostic pop +#endif /// Прочитать беззнаковое целое в простом формате из не-0-terminated строки. diff --git a/base/base/JSON.h b/base/base/JSON.h index 5965b2b17a9..3226a0d09e8 100644 --- a/base/base/JSON.h +++ b/base/base/JSON.h @@ -38,8 +38,14 @@ */ +#ifdef __clang__ +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec" +#endif POCO_DECLARE_EXCEPTION(Foundation_API, JSONException, Poco::Exception) - +#ifdef __clang__ +# pragma clang diagnostic pop +#endif class JSON { diff --git a/cmake/warnings.cmake b/cmake/warnings.cmake index 6b08f1fda05..0e55f6d32c3 100644 --- a/cmake/warnings.cmake +++ b/cmake/warnings.cmake @@ -20,16 +20,12 @@ if (COMPILER_CLANG) # We want to get everything out of the compiler for code quality. add_warning(everything) add_warning(pedantic) - no_warning(vla-extension) no_warning(zero-length-array) - no_warning(c11-extensions) - no_warning(unused-command-line-argument) no_warning(c++98-compat-pedantic) no_warning(c++98-compat) no_warning(c99-extensions) no_warning(conversion) no_warning(ctad-maybe-unsupported) # clang 9+, linux-only - no_warning(deprecated-dynamic-exception-spec) no_warning(disabled-macro-expansion) no_warning(documentation-unknown-command) no_warning(double-promotion) @@ -38,11 +34,7 @@ if (COMPILER_CLANG) no_warning(global-constructors) no_warning(missing-prototypes) no_warning(missing-variable-declarations) - no_warning(nested-anon-types) - no_warning(packed) no_warning(padded) - no_warning(return-std-move-in-c++11) # clang 7+ - no_warning(shift-sign-overflow) no_warning(sign-conversion) no_warning(switch-enum) no_warning(undefined-func-template) diff --git a/contrib/thrift b/contrib/thrift index 010ccf0a0c7..2a93df80f27 160000 --- a/contrib/thrift +++ b/contrib/thrift @@ -1 +1 @@ -Subproject commit 010ccf0a0c7023fea0f6bf4e4078ebdff7e61982 +Subproject commit 2a93df80f27739ccabb5b885cb12a8dc7595ecdf diff --git a/contrib/thrift-cmake/CMakeLists.txt b/contrib/thrift-cmake/CMakeLists.txt index 6f94c1ebdc0..d6aa6b9e5f2 100644 --- a/contrib/thrift-cmake/CMakeLists.txt +++ b/contrib/thrift-cmake/CMakeLists.txt @@ -15,7 +15,6 @@ set(thriftcpp_SOURCES "${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.cpp" "${LIBRARY_DIR}/src/thrift/concurrency/ThreadManager.cpp" "${LIBRARY_DIR}/src/thrift/concurrency/TimerManager.cpp" - "${LIBRARY_DIR}/src/thrift/concurrency/Util.cpp" "${LIBRARY_DIR}/src/thrift/processor/PeekProcessor.cpp" "${LIBRARY_DIR}/src/thrift/protocol/TBase64Utils.cpp" "${LIBRARY_DIR}/src/thrift/protocol/TDebugProtocol.cpp" @@ -33,6 +32,8 @@ set(thriftcpp_SOURCES "${LIBRARY_DIR}/src/thrift/transport/TServerSocket.cpp" "${LIBRARY_DIR}/src/thrift/transport/TTransportUtils.cpp" "${LIBRARY_DIR}/src/thrift/transport/TBufferTransports.cpp" + "${LIBRARY_DIR}/src/thrift/transport/SocketCommon.cpp" + "${LIBRARY_DIR}/src/thrift/transport/TWebSocketServer.cpp" "${LIBRARY_DIR}/src/thrift/server/TConnectedClient.cpp" "${LIBRARY_DIR}/src/thrift/server/TServerFramework.cpp" "${LIBRARY_DIR}/src/thrift/server/TSimpleServer.cpp" @@ -92,4 +93,4 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}") add_library(_thrift ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES}) add_library(ch_contrib::thrift ALIAS _thrift) target_include_directories(_thrift SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src" ${CMAKE_CURRENT_BINARY_DIR}) -target_link_libraries (_thrift PUBLIC boost::headers_only) +target_link_libraries (_thrift PUBLIC OpenSSL::SSL boost::headers_only) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 03a79b45a10..aa209f197df 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -71,6 +71,7 @@ RUN apt-get update \ software-properties-common \ tzdata \ unixodbc \ + file \ --yes --no-install-recommends RUN pip3 install numpy scipy pandas Jinja2 diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 96c06845812..9141e89d744 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -34,6 +34,7 @@ RUN apt-get update -y \ unixodbc \ wget \ zstd \ + file \ && apt-get clean diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index fcf99b34064..e34195cdd32 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -353,6 +353,9 @@ else # Error messages (we should ignore some errors) # FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.") + # FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64") + # FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server. + # Let's just ignore all errors from queries ("} TCPHandler: Code:", "} executeQuery: Code:") echo "Check for Error messages in server log:" zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \ -e "Code: 236. DB::Exception: Cancelled mutating parts" \ @@ -376,6 +379,9 @@ else -e "found in queue and some source parts for it was lost" \ -e "is lost forever." \ -e "Unknown index: idx." \ + -e "Cannot parse string 'Hello' as UInt64" \ + -e "} TCPHandler: Code:" \ + -e "} executeQuery: Code:" \ /var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "" > /test_output/bc_check_error_messages.txt \ && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv diff --git a/docs/en/operations/system-tables/part_log.md b/docs/en/operations/system-tables/part_log.md index 298667f895b..e1134f6baf6 100644 --- a/docs/en/operations/system-tables/part_log.md +++ b/docs/en/operations/system-tables/part_log.md @@ -19,6 +19,10 @@ The `system.part_log` table contains the following columns: - `REGULAR_MERGE` — Some regular merge. - `TTL_DELETE_MERGE` — Cleaning up expired data. - `TTL_RECOMPRESS_MERGE` — Recompressing data part with the. +- `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Merge algorithm for the event with type `MERGE_PARTS`. Can have one of the following values: + - `UNDECIDED` + - `HORIZONTAL` + - `VERTICAL` - `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time. - `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision. @@ -52,6 +56,7 @@ Row 1: query_id: 983ad9c7-28d5-4ae1-844e-603116b7de31 event_type: NewPart merge_reason: NotAMerge +merge_algorithm: Undecided event_date: 2021-02-02 event_time: 2021-02-02 11:14:28 event_time_microseconds: 2021-02-02 11:14:28.861919 diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index 53c2e929d15..50a394e45fd 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -1715,3 +1715,7 @@ Return value type is always [Float64](../../sql-reference/data-types/float.md). │ 6 │ Float64 │ └─────┴──────────────────────────────────────────────────────────────────────────────────────────┘ ``` + +## Distance functions + +All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md). diff --git a/docs/en/sql-reference/functions/distance-functions.md b/docs/en/sql-reference/functions/distance-functions.md new file mode 100644 index 00000000000..4af264f27ca --- /dev/null +++ b/docs/en/sql-reference/functions/distance-functions.md @@ -0,0 +1,502 @@ +# Distance functions + +## L1Norm + +Calculates the sum of absolute values of a vector. + +**Syntax** + +```sql +L1Norm(vector) +``` + +Alias: `normL1`. + +**Arguments** + +- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance. + +Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). + +**Examples** + +Query: + +```sql +SELECT L1Norm((1, 2)); +``` + +Result: + +```text +┌─L1Norm((1, 2))─┐ +│ 3 │ +└────────────────┘ +``` + +## L2Norm + +Calculates the square root of the sum of the squares of the vector values. + +**Syntax** + +```sql +L2Norm(vector) +``` + +Alias: `normL2`. + +**Arguments** + +- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance). + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT L2Norm((1, 2)); +``` + +Result: + +```text +┌───L2Norm((1, 2))─┐ +│ 2.23606797749979 │ +└──────────────────┘ +``` + +## LinfNorm + +Calculates the maximum of absolute values of a vector. + +**Syntax** + +```sql +LinfNorm(vector) +``` + +Alias: `normLinf`. + +**Arguments** + +- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- Linf-norm or the maximum absolute value. + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT LinfNorm((1, -2)); +``` + +Result: + +```text +┌─LinfNorm((1, -2))─┐ +│ 2 │ +└───────────────────┘ +``` + +## LpNorm + +Calculates the root of `p`-th power of the sum of the absolute values of a vector in the power of `p`. + +**Syntax** + +```sql +LpNorm(vector, p) +``` + +Alias: `normLp`. + +**Arguments** + +- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). +- `p` — The power. Possible values: real number in `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). + +**Returned value** + +- [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm) + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT LpNorm((1, -2), 2); +``` + +Result: + +```text +┌─LpNorm((1, -2), 2)─┐ +│ 2.23606797749979 │ +└────────────────────┘ +``` + +## L1Distance + +Calculates the distance between two points (the values of the vectors are the coordinates) in `L1` space (1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance)). + +**Syntax** + +```sql +L1Distance(vector1, vector2) +``` + +Alias: `distanceL1`. + +**Arguments** + +- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). +- `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- 1-norm distance. + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT L1Distance((1, 2), (2, 3)); +``` + +Result: + +```text +┌─L1Distance((1, 2), (2, 3))─┐ +│ 2 │ +└────────────────────────────┘ +``` + +## L2Distance + +Calculates the distance between two points (the values of the vectors are the coordinates) in Euclidean space ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)). + +**Syntax** + +```sql +L2Distance(vector1, vector2) +``` + +Alias: `distanceL2`. + +**Arguments** + +- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). +- `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- 2-norm distance. + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT L2Distance((1, 2), (2, 3)); +``` + +Result: + +```text +┌─L2Distance((1, 2), (2, 3))─┐ +│ 1.4142135623730951 │ +└────────────────────────────┘ +``` + +## LinfDistance + +Calculates the distance between two points (the values of the vectors are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))). + +**Syntax** + +```sql +LinfDistance(vector1, vector2) +``` + +Alias: `distanceLinf`. + +**Arguments** + +- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). +- `vector1` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). + +**Returned value** + +- Infinity-norm distance. + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT LinfDistance((1, 2), (2, 3)); +``` + +Result: + +```text +┌─LinfDistance((1, 2), (2, 3))─┐ +│ 1 │ +└──────────────────────────────┘ +``` + +## LpDistance + +Calculates the distance between two points (the values of the vectors are the coordinates) in `Lp` space ([p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)). + +**Syntax** + +```sql +LpDistance(vector1, vector2, p) +``` + +Alias: `distanceLp`. + +**Arguments** + +- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). +- `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md). +- `p` — The power. Possible values: real number from `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). + +**Returned value** + +- p-norm distance. + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT LpDistance((1, 2), (2, 3), 3); +``` + +Result: + +```text +┌─LpDistance((1, 2), (2, 3), 3)─┐ +│ 1.2599210498948732 │ +└───────────────────────────────┘ +``` + + +## L1Normalize + +Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L1` space ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)). + +**Syntax** + +```sql +L1Normalize(tuple) +``` + +Alias: `normalizeL1`. + +**Arguments** + +- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). + +**Returned value** + +- Unit vector. + +Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT L1Normalize((1, 2)); +``` + +Result: + +```text +┌─L1Normalize((1, 2))─────────────────────┐ +│ (0.3333333333333333,0.6666666666666666) │ +└─────────────────────────────────────────┘ +``` + +## L2Normalize + +Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in Euclidean space (using [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)). + +**Syntax** + +```sql +L2Normalize(tuple) +``` + +Alias: `normalizeL1`. + +**Arguments** + +- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). + +**Returned value** + +- Unit vector. + +Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT L2Normalize((3, 4)); +``` + +Result: + +```text +┌─L2Normalize((3, 4))─┐ +│ (0.6,0.8) │ +└─────────────────────┘ +``` + +## LinfNormalize + +Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L_{inf}` space (using [maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))). + +**Syntax** + +```sql +LinfNormalize(tuple) +``` + +Alias: `normalizeLinf `. + +**Arguments** + +- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). + +**Returned value** + +- Unit vector. + +Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT LinfNormalize((3, 4)); +``` + +Result: + +```text +┌─LinfNormalize((3, 4))─┐ +│ (0.75,1) │ +└───────────────────────┘ +``` + +## LpNormalize + +Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `Lp` space (using [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)). + +**Syntax** + +```sql +LpNormalize(tuple, p) +``` + +Alias: `normalizeLp `. + +**Arguments** + +- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). +- `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). + +**Returned value** + +- Unit vector. + +Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT LpNormalize((3, 4),5); +``` + +Result: + +```text +┌─LpNormalize((3, 4), 5)──────────────────┐ +│ (0.7187302630182624,0.9583070173576831) │ +└─────────────────────────────────────────┘ +``` + +## cosineDistance + +Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors. + +**Syntax** + +```sql +cosineDistance(tuple1, tuple2) +``` + +**Arguments** + +- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). +- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). + +**Returned value** + +- Cosine of the angle between two vectors substracted from one. + +Type: [Float](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +```sql +SELECT cosineDistance((1, 2), (2, 3)); +``` + +Result: + +```text +┌─cosineDistance((1, 2), (2, 3))─┐ +│ 0.007722123286332261 │ +└────────────────────────────────┘ +``` diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index 08fd979ce8f..3cd9bdf8ff5 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -93,3 +93,35 @@ Predefined characters: `\0`, `\\`, `|`, `(`, `)`, `^`, `$`, `.`, `[`, `]`, `?`, This implementation slightly differs from re2::RE2::QuoteMeta. It escapes zero byte as `\0` instead of `\x00` and it escapes only required characters. For more information, see the link: [RE2](https://github.com/google/re2/blob/master/re2/re2.cc#L473) + +## translate(s, from, to) + +The function replaces characters in the string ‘s’ in accordance with one-to-one character mapping defined by ‘from’ and ‘to’ strings. ‘from’ and ‘to’ must be ASCII strings of the same size. Non-ASCII characters in the original string are not modified. + +Example: + +``` sql +SELECT translate('Hello, World!', 'delor', 'DELOR') AS res +``` + +``` text +┌─res───────────┐ +│ HELLO, WORLD! │ +└───────────────┘ +``` + +## translateUTF8(string, from, to) + +Similar to previous function, but works with UTF-8 arguments. ‘from’ and ‘to’ must be valid UTF-8 strings of the same size. + +Example: + +``` sql +SELECT translateUTF8('Hélló, Wórld¡', 'óé¡', 'oe!') AS res +``` + +``` text +┌─res───────────┐ +│ Hello, World! │ +└───────────────┘ +``` diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 305e0c8c5cd..86408d8fc93 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -374,7 +374,7 @@ The same as `multiMatchAny`, but returns any index that matches the haystack. ## multiMatchAllIndices(haystack, \[pattern1, pattern2, …, patternn\]) -The same as `multiMatchAny`, but returns the array of all indicies that match the haystack in any order. +The same as `multiMatchAny`, but returns the array of all indices that match the haystack in any order. ## multiFuzzyMatchAny(haystack, distance, \[pattern1, pattern2, …, patternn\]) diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 64bdf9b9f33..d3aac635841 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -559,502 +559,7 @@ Result: └────────────────────────────┘ ``` -## L1Norm -Calculates the sum of absolute values of a tuple. +## Distance functions -**Syntax** - -```sql -L1Norm(tuple) -``` - -Alias: `normL1`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance. - -Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). - -**Example** - -Query: - -```sql -SELECT L1Norm((1, 2)); -``` - -Result: - -```text -┌─L1Norm((1, 2))─┐ -│ 3 │ -└────────────────┘ -``` - -## L2Norm - -Calculates the square root of the sum of the squares of the tuple values. - -**Syntax** - -```sql -L2Norm(tuple) -``` - -Alias: `normL2`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance). - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L2Norm((1, 2)); -``` - -Result: - -```text -┌───L2Norm((1, 2))─┐ -│ 2.23606797749979 │ -└──────────────────┘ -``` - -## LinfNorm - -Calculates the maximum of absolute values of a tuple. - -**Syntax** - -```sql -LinfNorm(tuple) -``` - -Alias: `normLinf`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Linf-norm or the maximum absolute value. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LinfNorm((1, -2)); -``` - -Result: - -```text -┌─LinfNorm((1, -2))─┐ -│ 2 │ -└───────────────────┘ -``` - -## LpNorm - -Calculates the root of `p`-th power of the sum of the absolute values of a tuple in the power of `p`. - -**Syntax** - -```sql -LpNorm(tuple, p) -``` - -Alias: `normLp`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). -- `p` — The power. Possible values: real number in `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). - -**Returned value** - -- [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm) - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LpNorm((1, -2), 2); -``` - -Result: - -```text -┌─LpNorm((1, -2), 2)─┐ -│ 2.23606797749979 │ -└────────────────────┘ -``` - -## L1Distance - -Calculates the distance between two points (the values of the tuples are the coordinates) in `L1` space (1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance)). - -**Syntax** - -```sql -L1Distance(tuple1, tuple2) -``` - -Alias: `distanceL1`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- 1-norm distance. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L1Distance((1, 2), (2, 3)); -``` - -Result: - -```text -┌─L1Distance((1, 2), (2, 3))─┐ -│ 2 │ -└────────────────────────────┘ -``` - -## L2Distance - -Calculates the distance between two points (the values of the tuples are the coordinates) in Euclidean space ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)). - -**Syntax** - -```sql -L2Distance(tuple1, tuple2) -``` - -Alias: `distanceL2`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- 2-norm distance. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L2Distance((1, 2), (2, 3)); -``` - -Result: - -```text -┌─L2Distance((1, 2), (2, 3))─┐ -│ 1.4142135623730951 │ -└────────────────────────────┘ -``` - -## LinfDistance - -Calculates the distance between two points (the values of the tuples are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))). - -**Syntax** - -```sql -LinfDistance(tuple1, tuple2) -``` - -Alias: `distanceLinf`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Infinity-norm distance. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LinfDistance((1, 2), (2, 3)); -``` - -Result: - -```text -┌─LinfDistance((1, 2), (2, 3))─┐ -│ 1 │ -└──────────────────────────────┘ -``` - -## LpDistance - -Calculates the distance between two points (the values of the tuples are the coordinates) in `Lp` space ([p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)). - -**Syntax** - -```sql -LpDistance(tuple1, tuple2, p) -``` - -Alias: `distanceLp`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `p` — The power. Possible values: real number from `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). - -**Returned value** - -- p-norm distance. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LpDistance((1, 2), (2, 3), 3); -``` - -Result: - -```text -┌─LpDistance((1, 2), (2, 3), 3)─┐ -│ 1.2599210498948732 │ -└───────────────────────────────┘ -``` - -## L1Normalize - -Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L1` space ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)). - -**Syntax** - -```sql -L1Normalize(tuple) -``` - -Alias: `normalizeL1`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Unit vector. - -Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L1Normalize((1, 2)); -``` - -Result: - -```text -┌─L1Normalize((1, 2))─────────────────────┐ -│ (0.3333333333333333,0.6666666666666666) │ -└─────────────────────────────────────────┘ -``` - -## L2Normalize - -Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in Euclidean space (using [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)). - -**Syntax** - -```sql -L2Normalize(tuple) -``` - -Alias: `normalizeL1`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Unit vector. - -Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L2Normalize((3, 4)); -``` - -Result: - -```text -┌─L2Normalize((3, 4))─┐ -│ (0.6,0.8) │ -└─────────────────────┘ -``` - -## LinfNormalize - -Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L_{inf}` space (using [maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))). - -**Syntax** - -```sql -LinfNormalize(tuple) -``` - -Alias: `normalizeLinf `. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Unit vector. - -Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LinfNormalize((3, 4)); -``` - -Result: - -```text -┌─LinfNormalize((3, 4))─┐ -│ (0.75,1) │ -└───────────────────────┘ -``` - -## LpNormalize - -Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `Lp` space (using [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)). - -**Syntax** - -```sql -LpNormalize(tuple, p) -``` - -Alias: `normalizeLp `. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). -- `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). - -**Returned value** - -- Unit vector. - -Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LpNormalize((3, 4),5); -``` - -Result: - -```text -┌─LpNormalize((3, 4), 5)──────────────────┐ -│ (0.7187302630182624,0.9583070173576831) │ -└─────────────────────────────────────────┘ -``` - -## cosineDistance - -Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors. - -**Syntax** - -```sql -cosineDistance(tuple1, tuple2) -``` - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Cosine of the angle between two vectors substracted from one. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT cosineDistance((1, 2), (2, 3)); -``` - -Result: - -```text -┌─cosineDistance((1, 2), (2, 3))─┐ -│ 0.007722123286332261 │ -└────────────────────────────────┘ -``` +All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md). diff --git a/docs/en/sql-reference/table-functions/null.md b/docs/en/sql-reference/table-functions/null.md index ae84705cb66..57a885afc54 100644 --- a/docs/en/sql-reference/table-functions/null.md +++ b/docs/en/sql-reference/table-functions/null.md @@ -5,7 +5,7 @@ sidebar_label: null function # null -Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately droped right after the query execution. The function is used for the convenience of test writing and demonstrations. +Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately dropped right after the query execution. The function is used for the convenience of test writing and demonstrations. **Syntax** @@ -40,4 +40,3 @@ See also: - [Null table engine](../../engines/table-engines/special/null.md) -[Original article](https://clickhouse.com/docs/en/sql-reference/table-functions/null/) diff --git a/docs/en/sql-reference/table-functions/remote.md b/docs/en/sql-reference/table-functions/remote.md index f9e6860b081..61018a3d5a7 100644 --- a/docs/en/sql-reference/table-functions/remote.md +++ b/docs/en/sql-reference/table-functions/remote.md @@ -5,11 +5,11 @@ sidebar_label: remote # remote, remoteSecure -Allows to access remote servers without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. `remoteSecure` - same as `remote` but with a secured connection. +Allows accessing remote servers, including migration of data, without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. `remoteSecure` - same as `remote` but with a secured connection. Both functions can be used in `SELECT` and `INSERT` queries. -**Syntax** +## Syntax ``` sql remote('addresses_expr', db, table[, 'user'[, 'password'], sharding_key]) @@ -18,7 +18,7 @@ remoteSecure('addresses_expr', db, table[, 'user'[, 'password'], sharding_key]) remoteSecure('addresses_expr', db.table[, 'user'[, 'password'], sharding_key]) ``` -**Parameters** +## Parameters - `addresses_expr` — An expression that generates addresses of remote servers. This may be just one server address. The server address is `host:port`, or just `host`. @@ -36,28 +36,31 @@ remoteSecure('addresses_expr', db.table[, 'user'[, 'password'], sharding_key]) - `password` — User password. If the password is not specified, an empty password is used. Type: [String](../../sql-reference/data-types/string.md). - `sharding_key` — Sharding key to support distributing data across nodes. For example: `insert into remote('127.0.0.1:9000,127.0.0.2', db, table, 'default', rand())`. Type: [UInt32](../../sql-reference/data-types/int-uint.md). -**Returned value** +## Returned value The dataset from remote servers. -**Usage** +## Usage -Using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function. +Unless you are migrating data from one system to another, using the `remote` table function is less optimal than creating a `Distributed` table because in this case the server connection is re-established for every request. Also, if hostnames are set, the names are resolved, and errors are not counted when working with various replicas. When processing a large number of queries, always create the `Distributed` table ahead of time, and do not use the `remote` table function. The `remote` table function can be useful in the following cases: -- Accessing a specific server for data comparison, debugging, and testing. -- Queries between various ClickHouse clusters for research purposes. -- Infrequent distributed requests that are made manually. -- Distributed requests where the set of servers is re-defined each time. +- Migrating data from one system to another +- Accessing a specific server for data comparison, debugging, and testing. +- Queries between various ClickHouse clusters for research purposes. +- Infrequent distributed requests that are made manually. +- Distributed requests where the set of servers is re-defined each time. -**Adresses** +### Adresses ``` text example01-01-1 +example01-01-1:9440 example01-01-1:9000 localhost 127.0.0.1 +[::]:9440 [::]:9000 [2a02:6b8:0:1111::11]:9000 ``` @@ -68,15 +71,15 @@ Multiple addresses can be comma-separated. In this case, ClickHouse will use dis example01-01-1,example01-02-1 ``` -**Examples** +## Examples -Selecting data from a remote server: +### Selecting data from a remote server: ``` sql SELECT * FROM remote('127.0.0.1', db.remote_engine_table) LIMIT 3; ``` -Inserting data from a remote server into a table: +### Inserting data from a remote server into a table: ``` sql CREATE TABLE remote_table (name String, value UInt32) ENGINE=Memory; @@ -84,7 +87,65 @@ INSERT INTO FUNCTION remote('127.0.0.1', currentDatabase(), 'remote_table') VALU SELECT * FROM remote_table; ``` -## Globs in Addresses {globs-in-addresses} +### Migration of tables from one system to another: +This example uses one table from a sample dataset. The database is `imdb`, and the table is `actors`. + +#### On the source ClickHouse system (the system that currently hosts the data) +- Verify the source database and table name (`imdb.actors`) + ```sql + show databases + ``` + + ```sql + show tables in imdb + ``` + +- Get the CREATE TABLE statement from the source: + ``` + select create_table_query + from system.tables + where database = 'imdb' and table = 'actors' + ``` + + Response + ```sql + CREATE TABLE imdb.actors (`id` UInt32, + `first_name` String, + `last_name` String, + `gender` FixedString(1)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') + ORDER BY (id, first_name, last_name, gender) + SETTINGS index_granularity = 8192 + ``` + +#### On the destination ClickHouse system: + +- Create the destination database: + ```sql + CREATE DATABASE imdb + ``` + +- Using the CREATE TABLE statement from the source, create the destination: + ```sql + CREATE TABLE imdb.actors (`id` UInt32, + `first_name` String, + `last_name` String, + `gender` FixedString(1)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') + ORDER BY (id, first_name, last_name, gender) + SETTINGS index_granularity = 8192 + ``` + +#### Back on the source deployment: + +Insert into the new database and table created on the remote system. You will need the host, port, username, password, destination database, and destination table. +```sql +INSERT INTO FUNCTION +remoteSecure('remote.clickhouse.cloud:9440', 'imdb.actors', 'USER', 'PASSWORD', rand()) +SELECT * from imdb.actors +``` + +## Globs in Addresses {#globs-in-addresses} Patterns in curly brackets `{ }` are used to generate a set of shards and to specify replicas. If there are multiple pairs of curly brackets, then the direct product of the corresponding sets is generated. The following pattern types are supported. diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 3ac79a5e718..e1e1b0b4268 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -335,7 +335,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 Поддерживаемые типы данных: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`. - Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem). + Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions#hasany), [hasAll](../../../sql-reference/functions/array-functions#hasall). **Примеры** diff --git a/docs/ru/operations/system-tables/part_log.md b/docs/ru/operations/system-tables/part_log.md index 78e9a7c0fbe..73ded8c7355 100644 --- a/docs/ru/operations/system-tables/part_log.md +++ b/docs/ru/operations/system-tables/part_log.md @@ -14,6 +14,15 @@ - `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition). - `MUTATE_PART` — изменение куска. - `MOVE_PART` — перемещение куска между дисками. +- `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — Причина события с типом `MERGE_PARTS`. Может принимать одно из следующих значений: + - `NOT_A_MERGE` — событие имеет тип иной, чем `MERGE_PARTS`. + - `REGULAR_MERGE` — обычное слияние. + - `TTL_DELETE_MERGE` — очистка истекших данных. + - `TTL_RECOMPRESS_MERGE` — переупаковка куска. +- `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Алгоритм слияния для события с типом `MERGE_PARTS`. Может принимать одно из следующих значений: + - `UNDECIDED` + - `HORIZONTAL` + - `VERTICAL` - `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата события. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время события. - `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время события с точностью до микросекунд. @@ -46,6 +55,8 @@ Row 1: ────── query_id: 983ad9c7-28d5-4ae1-844e-603116b7de31 event_type: NewPart +merge_reason: NotAMerge +merge_algorithm: Undecided event_date: 2021-02-02 event_time: 2021-02-02 11:14:28 event_time_microseconds: 2021-02-02 11:14:28.861919 diff --git a/docs/ru/sql-reference/functions/string-replace-functions.md b/docs/ru/sql-reference/functions/string-replace-functions.md index dbcdb2e902b..1bbb68f9014 100644 --- a/docs/ru/sql-reference/functions/string-replace-functions.md +++ b/docs/ru/sql-reference/functions/string-replace-functions.md @@ -83,3 +83,34 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res └─────────────────────┘ ``` +## translate(s, from, to) + +Данная функция заменяет символы в строке ‘s’ в соответствии с поэлементным отображением определяемым строками ‘from’ и ‘to’. ‘from’ и ‘to’ должны быть корректными ASCII строками одного размера. Не ASCII символы в оригинальной строке не изменяются. + +Example: + +``` sql +SELECT translate('Hello, World!', 'delor', 'DELOR') AS res +``` + +``` text +┌─res───────────┐ +│ HELLO, WORLD! │ +└───────────────┘ +``` + +## translateUTF8(string, from, to) + +Аналогично предыдущей функции, но работает со строками, состоящими из UTF-8 символов. ‘from’ и ‘to’ должны быть корректными UTF-8 строками одного размера. + +Example: + +``` sql +SELECT translateUTF8('Hélló, Wórld¡', 'óé¡', 'oe!') AS res +``` + +``` text +┌─res───────────┐ +│ Hello, World! │ +└───────────────┘ +``` diff --git a/programs/copier/ZooKeeperStaff.h b/programs/copier/ZooKeeperStaff.h index 66036ae2f27..a9e04578607 100644 --- a/programs/copier/ZooKeeperStaff.h +++ b/programs/copier/ZooKeeperStaff.h @@ -20,7 +20,7 @@ public: bool operator<=(const WrappingUInt32 & other) const { - const UInt32 HALF = 1 << 31; + const UInt32 HALF = static_cast(1) << 31; return (value <= other.value && other.value - value < HALF) || (value > other.value && value - other.value > HALF); } diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 7152820b5bc..abd481f0bb6 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -101,7 +101,7 @@ public: registered_prefixes = prefixes_; } - bool isSettingNameAllowed(const std::string_view & setting_name) const + bool isSettingNameAllowed(std::string_view setting_name) const { if (Settings::hasBuiltin(setting_name)) return true; @@ -116,7 +116,7 @@ public: return false; } - void checkSettingNameIsAllowed(const std::string_view & setting_name) const + void checkSettingNameIsAllowed(std::string_view setting_name) const { if (isSettingNameAllowed(setting_name)) return; diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 7c3139dbb0f..b6fed3ac912 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -252,7 +252,7 @@ public: } template - void grant(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames) + void grant(const AccessFlags & flags_, std::string_view name, const Args &... subnames) { auto & child = getChild(name); child.grant(flags_, subnames...); @@ -279,7 +279,7 @@ public: } template - void revoke(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames) + void revoke(const AccessFlags & flags_, std::string_view name, const Args &... subnames) { auto & child = getChild(name); @@ -306,7 +306,7 @@ public: } template - bool isGranted(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames) const + bool isGranted(const AccessFlags & flags_, std::string_view name, const Args &... subnames) const { AccessFlags flags_to_check = flags_ - min_flags_with_children; if (!flags_to_check) @@ -415,7 +415,7 @@ private: AccessFlags getAllGrantableFlags() const { return ::DB::getAllGrantableFlags(level); } AccessFlags getChildAllGrantableFlags() const { return ::DB::getAllGrantableFlags(static_cast(level + 1)); } - Node * tryGetChild(const std::string_view & name) const + Node * tryGetChild(std::string_view name) const { if (!children) return nullptr; @@ -425,7 +425,7 @@ private: return &it->second; } - Node & getChild(const std::string_view & name) + Node & getChild(std::string_view name) { auto * child = tryGetChild(name); if (child) @@ -819,20 +819,20 @@ void AccessRights::grantImpl(const AccessRightsElements & elements) } void AccessRights::grant(const AccessFlags & flags) { grantImpl(flags); } -void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl(flags, database); } -void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl(flags, database, table); } -void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl(flags, database, table, column); } -void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) { grantImpl(flags, database, table, columns); } -void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl(flags, database, table, columns); } +void AccessRights::grant(const AccessFlags & flags, std::string_view database) { grantImpl(flags, database); } +void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl(flags, database, table); } +void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl(flags, database, table, column); } +void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) { grantImpl(flags, database, table, columns); } +void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl(flags, database, table, columns); } void AccessRights::grant(const AccessRightsElement & element) { grantImpl(element); } void AccessRights::grant(const AccessRightsElements & elements) { grantImpl(elements); } void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl(flags); } -void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database) { grantImpl(flags, database); } -void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl(flags, database, table); } -void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl(flags, database, table, column); } -void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) { grantImpl(flags, database, table, columns); } -void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl(flags, database, table, columns); } +void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database) { grantImpl(flags, database); } +void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl(flags, database, table); } +void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl(flags, database, table, column); } +void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) { grantImpl(flags, database, table, columns); } +void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl(flags, database, table, columns); } void AccessRights::grantWithGrantOption(const AccessRightsElement & element) { grantImpl(element); } void AccessRights::grantWithGrantOption(const AccessRightsElements & elements) { grantImpl(elements); } @@ -892,20 +892,20 @@ void AccessRights::revokeImpl(const AccessRightsElements & elements) } void AccessRights::revoke(const AccessFlags & flags) { revokeImpl(flags); } -void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl(flags, database); } -void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl(flags, database, table); } -void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl(flags, database, table, column); } -void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) { revokeImpl(flags, database, table, columns); } -void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl(flags, database, table, columns); } +void AccessRights::revoke(const AccessFlags & flags, std::string_view database) { revokeImpl(flags, database); } +void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl(flags, database, table); } +void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl(flags, database, table, column); } +void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) { revokeImpl(flags, database, table, columns); } +void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl(flags, database, table, columns); } void AccessRights::revoke(const AccessRightsElement & element) { revokeImpl(element); } void AccessRights::revoke(const AccessRightsElements & elements) { revokeImpl(elements); } void AccessRights::revokeGrantOption(const AccessFlags & flags) { revokeImpl(flags); } -void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database) { revokeImpl(flags, database); } -void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl(flags, database, table); } -void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl(flags, database, table, column); } -void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) { revokeImpl(flags, database, table, columns); } -void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl(flags, database, table, columns); } +void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database) { revokeImpl(flags, database); } +void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl(flags, database, table); } +void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl(flags, database, table, column); } +void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) { revokeImpl(flags, database, table, columns); } +void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl(flags, database, table, columns); } void AccessRights::revokeGrantOption(const AccessRightsElement & element) { revokeImpl(element); } void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { revokeImpl(elements); } @@ -984,20 +984,20 @@ bool AccessRights::isGrantedImpl(const AccessRightsElements & elements) const } bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl(flags); } -bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl(flags, database); } -bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl(flags, database, table); } -bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl(flags, database, table, column); } -bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return isGrantedImpl(flags, database, table, columns); } -bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl(flags, database); } +bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl(flags, database, table); } +bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl(flags, database, table, column); } +bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl(flags, database, table, columns); } bool AccessRights::isGranted(const AccessRightsElement & element) const { return isGrantedImpl(element); } bool AccessRights::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl(elements); } bool AccessRights::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl(flags); } -bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl(flags, database); } -bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl(flags, database, table); } -bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl(flags, database, table, column); } -bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return isGrantedImpl(flags, database, table, columns); } -bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl(flags, database); } +bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl(flags, database, table); } +bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl(flags, database, table, column); } +bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl(flags, database, table, columns); } bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl(element); } bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl(elements); } diff --git a/src/Access/AccessRights.h b/src/Access/AccessRights.h index b7499d69f70..80e37561c2b 100644 --- a/src/Access/AccessRights.h +++ b/src/Access/AccessRights.h @@ -39,59 +39,59 @@ public: /// Grants access on a specified database/table/column. /// Does nothing if the specified access has been already granted. void grant(const AccessFlags & flags); - void grant(const AccessFlags & flags, const std::string_view & database); - void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table); - void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column); - void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns); - void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns); + void grant(const AccessFlags & flags, std::string_view database); + void grant(const AccessFlags & flags, std::string_view database, std::string_view table); + void grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column); + void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns); + void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns); void grant(const AccessRightsElement & element); void grant(const AccessRightsElements & elements); void grantWithGrantOption(const AccessFlags & flags); - void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database); - void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table); - void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column); - void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns); - void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns); + void grantWithGrantOption(const AccessFlags & flags, std::string_view database); + void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table); + void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column); + void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns); + void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns); void grantWithGrantOption(const AccessRightsElement & element); void grantWithGrantOption(const AccessRightsElements & elements); /// Revokes a specified access granted earlier on a specified database/table/column. /// For example, revoke(AccessType::ALL) revokes all grants at all, just like clear(); void revoke(const AccessFlags & flags); - void revoke(const AccessFlags & flags, const std::string_view & database); - void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table); - void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column); - void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns); - void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns); + void revoke(const AccessFlags & flags, std::string_view database); + void revoke(const AccessFlags & flags, std::string_view database, std::string_view table); + void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column); + void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns); + void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns); void revoke(const AccessRightsElement & element); void revoke(const AccessRightsElements & elements); void revokeGrantOption(const AccessFlags & flags); - void revokeGrantOption(const AccessFlags & flags, const std::string_view & database); - void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table); - void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column); - void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns); - void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns); + void revokeGrantOption(const AccessFlags & flags, std::string_view database); + void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table); + void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column); + void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns); + void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns); void revokeGrantOption(const AccessRightsElement & element); void revokeGrantOption(const AccessRightsElements & elements); /// Whether a specified access granted. bool isGranted(const AccessFlags & flags) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + bool isGranted(const AccessFlags & flags, std::string_view database) const; + bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const; + bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const; + bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const; + bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const; bool isGranted(const AccessRightsElement & element) const; bool isGranted(const AccessRightsElements & elements) const; bool hasGrantOption(const AccessFlags & flags) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + bool hasGrantOption(const AccessFlags & flags, std::string_view database) const; + bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const; + bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const; + bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const; + bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const; bool hasGrantOption(const AccessRightsElement & element) const; bool hasGrantOption(const AccessRightsElements & elements) const; @@ -107,9 +107,9 @@ public: const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, - const std::string_view & database, - const std::string_view & table, - const std::string_view & column)>; + std::string_view database, + std::string_view table, + std::string_view column)>; void modifyFlags(const ModifyFlagsFunction & function); void modifyFlagsWithGrantOption(const ModifyFlagsFunction & function); diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 4f304cf5952..c6bbd421c77 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -26,17 +26,17 @@ namespace return (Util::stringToDigest(password) == password_plaintext); } - bool checkPasswordDoubleSHA1(const std::string_view & password, const Digest & password_double_sha1) + bool checkPasswordDoubleSHA1(std::string_view password, const Digest & password_double_sha1) { return (Util::encodeDoubleSHA1(password) == password_double_sha1); } - bool checkPasswordSHA256(const std::string_view & password, const Digest & password_sha256, const String & salt) + bool checkPasswordSHA256(std::string_view password, const Digest & password_sha256, const String & salt) { return Util::encodeSHA256(String(password).append(salt)) == password_sha256; } - bool checkPasswordDoubleSHA1MySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_double_sha1) + bool checkPasswordDoubleSHA1MySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_double_sha1) { /// scrambled_password = SHA1(password) XOR SHA1(scramble SHA1(SHA1(password))) @@ -61,7 +61,7 @@ namespace return calculated_password_double_sha1 == password_double_sha1; } - bool checkPasswordPlainTextMySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_plaintext) + bool checkPasswordPlainTextMySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_plaintext) { return checkPasswordDoubleSHA1MySQL(scramble, scrambled_password, Util::encodeDoubleSHA1(password_plaintext)); } diff --git a/src/Access/Common/AccessFlags.cpp b/src/Access/Common/AccessFlags.cpp index 82e1cbfb26b..305ae3f7cf5 100644 --- a/src/Access/Common/AccessFlags.cpp +++ b/src/Access/Common/AccessFlags.cpp @@ -35,7 +35,7 @@ namespace return access_type_to_flags_mapping[static_cast(type)]; } - Flags keywordToFlags(const std::string_view & keyword) const + Flags keywordToFlags(std::string_view keyword) const { auto it = keyword_to_flags_map.find(keyword); if (it == keyword_to_flags_map.end()) @@ -142,14 +142,14 @@ namespace } }; - static String replaceUnderscoreWithSpace(const std::string_view & str) + static String replaceUnderscoreWithSpace(std::string_view str) { String res{str}; boost::replace_all(res, "_", " "); return res; } - static Strings splitAliases(const std::string_view & str) + static Strings splitAliases(std::string_view str) { Strings aliases; boost::split(aliases, str, boost::is_any_of(",")); @@ -160,10 +160,10 @@ namespace static void makeNode( AccessType access_type, - const std::string_view & name, - const std::string_view & aliases, + std::string_view name, + std::string_view aliases, NodeType node_type, - const std::string_view & parent_group_name, + std::string_view parent_group_name, std::unordered_map & nodes, std::unordered_map & owned_nodes, size_t & next_flag) @@ -353,7 +353,7 @@ namespace AccessFlags::AccessFlags(AccessType type) : flags(Helper::instance().accessTypeToFlags(type)) {} -AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Helper::instance().keywordToFlags(keyword)) {} +AccessFlags::AccessFlags(std::string_view keyword) : flags(Helper::instance().keywordToFlags(keyword)) {} AccessFlags::AccessFlags(const std::vector & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {} AccessFlags::AccessFlags(const Strings & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {} String AccessFlags::toString() const { return Helper::instance().flagsToString(flags); } diff --git a/src/Access/Common/AccessFlags.h b/src/Access/Common/AccessFlags.h index 51bf3cd19b0..5124f4ef332 100644 --- a/src/Access/Common/AccessFlags.h +++ b/src/Access/Common/AccessFlags.h @@ -21,7 +21,7 @@ public: AccessFlags() = default; /// Constructs from a string like "SELECT". - AccessFlags(const std::string_view & keyword); /// NOLINT + AccessFlags(std::string_view keyword); /// NOLINT /// Constructs from a list of strings like "SELECT, UPDATE, INSERT". AccessFlags(const std::vector & keywords); /// NOLINT diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp index 9913fc02f4a..69a2354f25d 100644 --- a/src/Access/Common/AccessRightsElement.cpp +++ b/src/Access/Common/AccessRightsElement.cpp @@ -81,7 +81,7 @@ namespace } bool need_comma = false; - for (const std::string_view & keyword : keywords) + for (std::string_view keyword : keywords) { if (need_comma) result.append(", "); @@ -145,18 +145,18 @@ namespace } -AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_) +AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_) : access_flags(access_flags_), database(database_), any_database(false) { } -AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_) +AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_) : access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false) { } AccessRightsElement::AccessRightsElement( - AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_) + AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_) : access_flags(access_flags_) , database(database_) , table(table_) @@ -169,8 +169,8 @@ AccessRightsElement::AccessRightsElement( AccessRightsElement::AccessRightsElement( AccessFlags access_flags_, - const std::string_view & database_, - const std::string_view & table_, + std::string_view database_, + std::string_view table_, const std::vector & columns_) : access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false), any_column(false) { @@ -180,7 +180,7 @@ AccessRightsElement::AccessRightsElement( } AccessRightsElement::AccessRightsElement( - AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_) + AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_) : access_flags(access_flags_) , database(database_) , table(table_) diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h index e5bf76d0017..5f65b6bcd12 100644 --- a/src/Access/Common/AccessRightsElement.h +++ b/src/Access/Common/AccessRightsElement.h @@ -28,19 +28,19 @@ struct AccessRightsElement explicit AccessRightsElement(AccessFlags access_flags_) : access_flags(access_flags_) {} - AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_); - AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_); + AccessRightsElement(AccessFlags access_flags_, std::string_view database_); + AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_); AccessRightsElement( - AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_); + AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_); AccessRightsElement( AccessFlags access_flags_, - const std::string_view & database_, - const std::string_view & table_, + std::string_view database_, + std::string_view table_, const std::vector & columns_); AccessRightsElement( - AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_); + AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_); bool empty() const { return !access_flags || (!any_column && columns.empty()); } diff --git a/src/Access/Common/AccessType.cpp b/src/Access/Common/AccessType.cpp index d44d70d78b2..4df1e1bc77f 100644 --- a/src/Access/Common/AccessType.cpp +++ b/src/Access/Common/AccessType.cpp @@ -35,7 +35,7 @@ namespace #undef ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING } - void addToMapping(AccessType type, const std::string_view & str) + void addToMapping(AccessType type, std::string_view str) { String str2{str}; boost::replace_all(str2, "_", " "); diff --git a/src/Access/Common/AuthenticationData.cpp b/src/Access/Common/AuthenticationData.cpp index db0a5d54a63..f3d3bb5b758 100644 --- a/src/Access/Common/AuthenticationData.cpp +++ b/src/Access/Common/AuthenticationData.cpp @@ -71,7 +71,7 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty } -AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::string_view & text [[maybe_unused]]) +AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_view text [[maybe_unused]]) { #if USE_SSL Digest hash; @@ -86,7 +86,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::str } -AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(const std::string_view & text) +AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(std::string_view text) { Poco::SHA1Engine engine; engine.update(text.data(), text.size()); diff --git a/src/Access/Common/AuthenticationData.h b/src/Access/Common/AuthenticationData.h index 2837e0f10a1..ced9fcd4b6d 100644 --- a/src/Access/Common/AuthenticationData.h +++ b/src/Access/Common/AuthenticationData.h @@ -96,11 +96,11 @@ public: struct Util { - static Digest stringToDigest(const std::string_view & text) { return Digest(text.data(), text.data() + text.size()); } - static Digest encodeSHA256(const std::string_view & text); - static Digest encodeSHA1(const std::string_view & text); + static Digest stringToDigest(std::string_view text) { return Digest(text.data(), text.data() + text.size()); } + static Digest encodeSHA256(std::string_view text); + static Digest encodeSHA1(std::string_view text); static Digest encodeSHA1(const Digest & text) { return encodeSHA1(std::string_view{reinterpret_cast(text.data()), text.size()}); } - static Digest encodeDoubleSHA1(const std::string_view & text) { return encodeSHA1(encodeSHA1(text)); } + static Digest encodeDoubleSHA1(std::string_view text) { return encodeSHA1(encodeSHA1(text)); } static Digest encodeDoubleSHA1(const Digest & text) { return encodeSHA1(encodeSHA1(text)); } }; diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 995a46d07ca..92a5179d861 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -46,7 +46,7 @@ namespace AccessRights addImplicitAccessRights(const AccessRights & access) { - auto modifier = [&](const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, const std::string_view & database, const std::string_view & table, const std::string_view & column) -> AccessFlags + auto modifier = [&](const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, std::string_view database, std::string_view table, std::string_view column) -> AccessFlags { size_t level = !database.empty() + !table.empty() + !column.empty(); AccessFlags res = flags; @@ -141,7 +141,7 @@ namespace std::string_view getDatabase() { return {}; } template - std::string_view getDatabase(const std::string_view & arg1, const OtherArgs &...) { return arg1; } + std::string_view getDatabase(std::string_view arg1, const OtherArgs &...) { return arg1; } } @@ -519,7 +519,7 @@ bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const } template -bool ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const +bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const { return checkAccessImplHelper(flags, database.empty() ? params.current_database : database, args...); } @@ -564,38 +564,38 @@ bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const } bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl(flags); } -bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl(flags, database); } -bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl(flags, database, table); } -bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl(flags, database, table, column); } -bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return checkAccessImpl(flags, database, table, columns); } -bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl(flags, database, table, columns); } +bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl(flags, database); } +bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl(flags, database, table); } +bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl(flags, database, table, column); } +bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const { return checkAccessImpl(flags, database, table, columns); } +bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl(flags, database, table, columns); } bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl(element); } bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl(elements); } bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return checkAccessImpl(flags); } -bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl(flags, database); } -bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl(flags, database, table); } -bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl(flags, database, table, column); } -bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return checkAccessImpl(flags, database, table, columns); } -bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl(flags, database, table, columns); } +bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl(flags, database); } +bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl(flags, database, table); } +bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl(flags, database, table, column); } +bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const { return checkAccessImpl(flags, database, table, columns); } +bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl(flags, database, table, columns); } bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return checkAccessImpl(element); } bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return checkAccessImpl(elements); } void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl(flags); } -void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl(flags, database); } -void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl(flags, database, table); } -void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl(flags, database, table, column); } -void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { checkAccessImpl(flags, database, table, columns); } -void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl(flags, database, table, columns); } +void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database) const { checkAccessImpl(flags, database); } +void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl(flags, database, table); } +void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl(flags, database, table, column); } +void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const { checkAccessImpl(flags, database, table, columns); } +void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl(flags, database, table, columns); } void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl(element); } void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl(elements); } void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl(flags); } -void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl(flags, database); } -void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl(flags, database, table); } -void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl(flags, database, table, column); } -void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { checkAccessImpl(flags, database, table, columns); } -void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl(flags, database, table, columns); } +void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database) const { checkAccessImpl(flags, database); } +void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl(flags, database, table); } +void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl(flags, database, table, column); } +void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const { checkAccessImpl(flags, database, table, columns); } +void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl(flags, database, table, columns); } void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl(element); } void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl(elements); } diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index 729574898aa..fa3523977e7 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -101,40 +101,40 @@ public: /// Checks if a specified access is granted, and throws an exception if not. /// Empty database means the current database. void checkAccess(const AccessFlags & flags) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + void checkAccess(const AccessFlags & flags, std::string_view database) const; + void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const; + void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const; + void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const; + void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const; void checkAccess(const AccessRightsElement & element) const; void checkAccess(const AccessRightsElements & elements) const; void checkGrantOption(const AccessFlags & flags) const; - void checkGrantOption(const AccessFlags & flags, const std::string_view & database) const; - void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + void checkGrantOption(const AccessFlags & flags, std::string_view database) const; + void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const; + void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const; + void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const; + void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const; void checkGrantOption(const AccessRightsElement & element) const; void checkGrantOption(const AccessRightsElements & elements) const; /// Checks if a specified access is granted, and returns false if not. /// Empty database means the current database. bool isGranted(const AccessFlags & flags) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + bool isGranted(const AccessFlags & flags, std::string_view database) const; + bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const; + bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const; + bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const; + bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const; bool isGranted(const AccessRightsElement & element) const; bool isGranted(const AccessRightsElements & elements) const; bool hasGrantOption(const AccessFlags & flags) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + bool hasGrantOption(const AccessFlags & flags, std::string_view database) const; + bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const; + bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const; + bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const; + bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const; bool hasGrantOption(const AccessRightsElement & element) const; bool hasGrantOption(const AccessRightsElements & elements) const; @@ -180,7 +180,7 @@ private: bool checkAccessImpl(const AccessFlags & flags) const; template - bool checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const; + bool checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const; template bool checkAccessImpl(const AccessRightsElement & element) const; diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 6084138f306..34f2e10dc83 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -36,12 +36,12 @@ void SettingsConstraints::clear() } -void SettingsConstraints::setMinValue(const std::string_view & setting_name, const Field & min_value) +void SettingsConstraints::setMinValue(std::string_view setting_name, const Field & min_value) { getConstraintRef(setting_name).min_value = Settings::castValueUtil(setting_name, min_value); } -Field SettingsConstraints::getMinValue(const std::string_view & setting_name) const +Field SettingsConstraints::getMinValue(std::string_view setting_name) const { const auto * ptr = tryGetConstraint(setting_name); if (ptr) @@ -51,12 +51,12 @@ Field SettingsConstraints::getMinValue(const std::string_view & setting_name) co } -void SettingsConstraints::setMaxValue(const std::string_view & setting_name, const Field & max_value) +void SettingsConstraints::setMaxValue(std::string_view setting_name, const Field & max_value) { getConstraintRef(setting_name).max_value = Settings::castValueUtil(setting_name, max_value); } -Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) const +Field SettingsConstraints::getMaxValue(std::string_view setting_name) const { const auto * ptr = tryGetConstraint(setting_name); if (ptr) @@ -66,12 +66,12 @@ Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) co } -void SettingsConstraints::setReadOnly(const std::string_view & setting_name, bool read_only) +void SettingsConstraints::setReadOnly(std::string_view setting_name, bool read_only) { getConstraintRef(setting_name).read_only = read_only; } -bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) const +bool SettingsConstraints::isReadOnly(std::string_view setting_name) const { const auto * ptr = tryGetConstraint(setting_name); if (ptr) @@ -81,7 +81,7 @@ bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) cons } -void SettingsConstraints::set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only) +void SettingsConstraints::set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only) { auto & ref = getConstraintRef(setting_name); ref.min_value = Settings::castValueUtil(setting_name, min_value); @@ -89,7 +89,7 @@ void SettingsConstraints::set(const std::string_view & setting_name, const Field ref.read_only = read_only; } -void SettingsConstraints::get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const +void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const { const auto * ptr = tryGetConstraint(setting_name); if (ptr) @@ -318,7 +318,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh } -SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const std::string_view & setting_name) +SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(std::string_view setting_name) { auto it = constraints.find(setting_name); if (it == constraints.end()) @@ -331,7 +331,7 @@ SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const st return it->second; } -const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(const std::string_view & setting_name) const +const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(std::string_view setting_name) const { auto it = constraints.find(setting_name); if (it == constraints.end()) diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index f7bca1eafb3..645a690e051 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -61,17 +61,17 @@ public: void clear(); bool empty() const { return constraints.empty(); } - void setMinValue(const std::string_view & setting_name, const Field & min_value); - Field getMinValue(const std::string_view & setting_name) const; + void setMinValue(std::string_view setting_name, const Field & min_value); + Field getMinValue(std::string_view setting_name) const; - void setMaxValue(const std::string_view & setting_name, const Field & max_value); - Field getMaxValue(const std::string_view & setting_name) const; + void setMaxValue(std::string_view setting_name, const Field & max_value); + Field getMaxValue(std::string_view setting_name) const; - void setReadOnly(const std::string_view & setting_name, bool read_only); - bool isReadOnly(const std::string_view & setting_name) const; + void setReadOnly(std::string_view setting_name, bool read_only); + bool isReadOnly(std::string_view setting_name) const; - void set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only); - void get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const; + void set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only); + void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const; void merge(const SettingsConstraints & other); @@ -105,8 +105,8 @@ private: }; bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const; - Constraint & getConstraintRef(const std::string_view & setting_name); - const Constraint * tryGetConstraint(const std::string_view & setting_name) const; + Constraint & getConstraintRef(std::string_view setting_name); + const Constraint * tryGetConstraint(std::string_view setting_name) const; std::unordered_map constraints; const AccessControl * access_control = nullptr; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ac68542684e..2424f2c42d8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -401,11 +401,11 @@ dbms_target_link_libraries ( clickhouse_parsers ch_contrib::lz4 Poco::JSON - Poco::MongoDB string_utils PUBLIC boost::system clickhouse_common_io + Poco::MongoDB ) if (TARGET ch::mysqlxx) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 0e243f97aaf..b435b483b71 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -601,6 +601,7 @@ void ClientBase::initLogsOutputStream() { WriteBuffer * wb = out_logs_buf.get(); + bool color_logs = false; if (!out_logs_buf) { if (server_logs_file.empty()) @@ -608,11 +609,13 @@ void ClientBase::initLogsOutputStream() /// Use stderr by default out_logs_buf = std::make_unique(STDERR_FILENO); wb = out_logs_buf.get(); + color_logs = stderr_is_a_tty; } else if (server_logs_file == "-") { /// Use stdout if --server_logs_file=- specified wb = &std_out; + color_logs = stdout_is_a_tty; } else { @@ -622,7 +625,7 @@ void ClientBase::initLogsOutputStream() } } - logs_out_stream = std::make_unique(*wb, stdout_is_a_tty); + logs_out_stream = std::make_unique(*wb, color_logs); } } @@ -2155,6 +2158,7 @@ void ClientBase::init(int argc, char ** argv) stdin_is_a_tty = isatty(STDIN_FILENO); stdout_is_a_tty = isatty(STDOUT_FILENO); + stderr_is_a_tty = isatty(STDERR_FILENO); terminal_width = getTerminalWidth(); Arguments common_arguments{""}; /// 0th argument is ignored. diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index ec2267a3be6..b012680fc3c 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -173,6 +173,7 @@ protected: bool stdin_is_a_tty = false; /// stdin is a terminal. bool stdout_is_a_tty = false; /// stdout is a terminal. + bool stderr_is_a_tty = false; /// stderr is a terminal. uint64_t terminal_width = 0; ServerConnectionPtr connection; diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index 750aa6a1cb2..93cbf269a8e 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include namespace Poco { class Logger; } @@ -33,7 +33,7 @@ friend class FileSegmentRangeWriter; public: using Key = IFileCache::Key; - using RemoteFileReaderPtr = std::shared_ptr; + using RemoteFileReaderPtr = std::shared_ptr; using LocalCacheWriterPtr = std::unique_ptr; enum class State diff --git a/src/Common/JSONParsers/DummyJSONParser.h b/src/Common/JSONParsers/DummyJSONParser.h index 77b958d1429..3cedd59decd 100644 --- a/src/Common/JSONParsers/DummyJSONParser.h +++ b/src/Common/JSONParsers/DummyJSONParser.h @@ -84,7 +84,7 @@ struct DummyJSONParser static Iterator begin() { return {}; } static Iterator end() { return {}; } static size_t size() { return 0; } - bool find(const std::string_view &, Element &) const { return false; } /// NOLINT + bool find(std::string_view, Element &) const { return false; } /// NOLINT #if 0 /// Optional: Provides access to an object's element by index. @@ -93,7 +93,7 @@ struct DummyJSONParser }; /// Parses a JSON document, returns the reference to its root element if succeeded. - bool parse(const std::string_view &, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT + bool parse(std::string_view, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT #if 0 /// Optional: Allocates memory to parse JSON documents faster. diff --git a/src/Common/JSONParsers/RapidJSONParser.h b/src/Common/JSONParsers/RapidJSONParser.h index 2d8514868e5..77e8f6b2a74 100644 --- a/src/Common/JSONParsers/RapidJSONParser.h +++ b/src/Common/JSONParsers/RapidJSONParser.h @@ -98,7 +98,7 @@ struct RapidJSONParser ALWAYS_INLINE Iterator end() const { return ptr->MemberEnd(); } ALWAYS_INLINE size_t size() const { return ptr->MemberCount(); } - bool find(const std::string_view & key, Element & result) const + bool find(std::string_view key, Element & result) const { auto it = ptr->FindMember(rapidjson::StringRef(key.data(), key.length())); if (it == ptr->MemberEnd()) @@ -122,7 +122,7 @@ struct RapidJSONParser }; /// Parses a JSON document, returns the reference to its root element if succeeded. - bool parse(const std::string_view & json, Element & result) + bool parse(std::string_view json, Element & result) { rapidjson::MemoryStream ms(json.data(), json.size()); rapidjson::EncodedInputStream, rapidjson::MemoryStream> is(ms); diff --git a/src/Common/JSONParsers/SimdJSONParser.h b/src/Common/JSONParsers/SimdJSONParser.h index 3abeb85fb56..f3bbfe4dfde 100644 --- a/src/Common/JSONParsers/SimdJSONParser.h +++ b/src/Common/JSONParsers/SimdJSONParser.h @@ -105,7 +105,7 @@ struct SimdJSONParser ALWAYS_INLINE Iterator end() const { return object.end(); } ALWAYS_INLINE size_t size() const { return object.size(); } - bool find(const std::string_view & key, Element & result) const + bool find(std::string_view key, Element & result) const { auto x = object.at_key(key); if (x.error()) @@ -131,7 +131,7 @@ struct SimdJSONParser }; /// Parses a JSON document, returns the reference to its root element if succeeded. - bool parse(const std::string_view & json, Element & result) + bool parse(std::string_view json, Element & result) { auto document = parser.parse(json.data(), json.size()); if (document.error()) diff --git a/src/Common/OpenSSLHelpers.cpp b/src/Common/OpenSSLHelpers.cpp index d73e08e79a9..4e7848afc85 100644 --- a/src/Common/OpenSSLHelpers.cpp +++ b/src/Common/OpenSSLHelpers.cpp @@ -10,7 +10,7 @@ namespace DB { #pragma GCC diagnostic warning "-Wold-style-cast" -std::string encodeSHA256(const std::string_view & text) +std::string encodeSHA256(std::string_view text) { return encodeSHA256(text.data(), text.size()); } @@ -21,7 +21,7 @@ std::string encodeSHA256(const void * text, size_t size) encodeSHA256(text, size, reinterpret_cast(out.data())); return out; } -void encodeSHA256(const std::string_view & text, unsigned char * out) +void encodeSHA256(std::string_view text, unsigned char * out) { encodeSHA256(text.data(), text.size(), out); } diff --git a/src/Common/OpenSSLHelpers.h b/src/Common/OpenSSLHelpers.h index f0dbbc10b4c..41f092f0109 100644 --- a/src/Common/OpenSSLHelpers.h +++ b/src/Common/OpenSSLHelpers.h @@ -10,10 +10,10 @@ namespace DB { /// Encodes `text` and returns it. -std::string encodeSHA256(const std::string_view & text); +std::string encodeSHA256(std::string_view text); std::string encodeSHA256(const void * text, size_t size); /// `out` must be at least 32 bytes long. -void encodeSHA256(const std::string_view & text, unsigned char * out); +void encodeSHA256(std::string_view text, unsigned char * out); void encodeSHA256(const void * text, size_t size, unsigned char * out); /// Returns concatenation of error strings for all errors that OpenSSL has recorded, emptying the error queue. diff --git a/src/Common/SettingsChanges.cpp b/src/Common/SettingsChanges.cpp index 370b465eba3..9fb4f361e09 100644 --- a/src/Common/SettingsChanges.cpp +++ b/src/Common/SettingsChanges.cpp @@ -4,7 +4,7 @@ namespace DB { namespace { - SettingChange * find(SettingsChanges & changes, const std::string_view & name) + SettingChange * find(SettingsChanges & changes, std::string_view name) { auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; }); if (it == changes.end()) @@ -12,7 +12,7 @@ namespace return &*it; } - const SettingChange * find(const SettingsChanges & changes, const std::string_view & name) + const SettingChange * find(const SettingsChanges & changes, std::string_view name) { auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; }); if (it == changes.end()) @@ -21,7 +21,7 @@ namespace } } -bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) const +bool SettingsChanges::tryGet(std::string_view name, Field & out_value) const { const auto * change = find(*this, name); if (!change) @@ -30,7 +30,7 @@ bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) c return true; } -const Field * SettingsChanges::tryGet(const std::string_view & name) const +const Field * SettingsChanges::tryGet(std::string_view name) const { const auto * change = find(*this, name); if (!change) @@ -38,7 +38,7 @@ const Field * SettingsChanges::tryGet(const std::string_view & name) const return &change->value; } -Field * SettingsChanges::tryGet(const std::string_view & name) +Field * SettingsChanges::tryGet(std::string_view name) { auto * change = find(*this, name); if (!change) diff --git a/src/Common/SettingsChanges.h b/src/Common/SettingsChanges.h index 5f6a390d0d2..67cb69f77bf 100644 --- a/src/Common/SettingsChanges.h +++ b/src/Common/SettingsChanges.h @@ -14,8 +14,8 @@ struct SettingChange Field value; SettingChange() = default; - SettingChange(const std::string_view & name_, const Field & value_) : name(name_), value(value_) {} - SettingChange(const std::string_view & name_, Field && value_) : name(name_), value(std::move(value_)) {} + SettingChange(std::string_view name_, const Field & value_) : name(name_), value(value_) {} + SettingChange(std::string_view name_, Field && value_) : name(name_), value(std::move(value_)) {} friend bool operator ==(const SettingChange & lhs, const SettingChange & rhs) { return (lhs.name == rhs.name) && (lhs.value == rhs.value); } friend bool operator !=(const SettingChange & lhs, const SettingChange & rhs) { return !(lhs == rhs); } @@ -27,9 +27,9 @@ class SettingsChanges : public std::vector public: using std::vector::vector; - bool tryGet(const std::string_view & name, Field & out_value) const; - const Field * tryGet(const std::string_view & name) const; - Field * tryGet(const std::string_view & name); + bool tryGet(std::string_view name, Field & out_value) const; + const Field * tryGet(std::string_view name) const; + Field * tryGet(std::string_view name); }; } diff --git a/src/Common/StringUtils/StringUtils.h b/src/Common/StringUtils/StringUtils.h index 21df0f5ae8b..b5a081ab693 100644 --- a/src/Common/StringUtils/StringUtils.h +++ b/src/Common/StringUtils/StringUtils.h @@ -147,7 +147,7 @@ inline bool isPunctuationASCII(char c) } -inline bool isValidIdentifier(const std::string_view & str) +inline bool isValidIdentifier(std::string_view str) { return !str.empty() && isValidIdentifierBegin(str[0]) diff --git a/src/Common/TaskStatsInfoGetter.cpp b/src/Common/TaskStatsInfoGetter.cpp index 36e8a0fce00..304ccc84765 100644 --- a/src/Common/TaskStatsInfoGetter.cpp +++ b/src/Common/TaskStatsInfoGetter.cpp @@ -21,6 +21,7 @@ #if defined(__clang__) #pragma clang diagnostic ignored "-Wgnu-anonymous-struct" + #pragma clang diagnostic ignored "-Wnested-anon-types" #endif /// Basic idea is motivated by "iotop" tool. diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index 4bd1cf391ef..eb3f631b92a 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -162,20 +162,19 @@ public: template explicit ThreadFromGlobalPool(Function && func, Args &&... args) - : state(std::make_shared()) - , thread_id(std::make_shared()) + : state(std::make_shared()) { - /// NOTE: If this will throw an exception, the destructor won't be called. + /// NOTE: + /// - If this will throw an exception, the destructor won't be called + /// - this pointer cannot be passed in the lambda, since after detach() it will not be valid GlobalThreadPool::instance().scheduleOrThrow([ - thread_id = thread_id, state = state, func = std::forward(func), args = std::make_tuple(std::forward(args)...)]() mutable /// mutable is needed to destroy capture { - auto event = std::move(state); - SCOPE_EXIT(event->set()); + SCOPE_EXIT(state->event.set()); - thread_id = std::make_shared(std::this_thread::get_id()); + state->thread_id = std::this_thread::get_id(); /// This moves are needed to destroy function and arguments before exit. /// It will guarantee that after ThreadFromGlobalPool::join all captured params are destroyed. @@ -196,31 +195,30 @@ public: ThreadFromGlobalPool & operator=(ThreadFromGlobalPool && rhs) noexcept { - if (joinable()) + if (initialized()) abort(); state = std::move(rhs.state); - thread_id = std::move(rhs.thread_id); return *this; } ~ThreadFromGlobalPool() { - if (joinable()) + if (initialized()) abort(); } void join() { - if (!joinable()) + if (!initialized()) abort(); - state->wait(); + state->event.wait(); state.reset(); } void detach() { - if (!joinable()) + if (!initialized()) abort(); state.reset(); } @@ -230,15 +228,30 @@ public: if (!state) return false; /// Thread cannot join itself. - if (*thread_id == std::this_thread::get_id()) + if (state->thread_id == std::this_thread::get_id()) return false; return true; } private: - /// The state used in this object and inside the thread job. - std::shared_ptr state; - std::shared_ptr thread_id; + struct State + { + /// Should be atomic() because of possible concurrent access between + /// assignment and joinable() check. + std::atomic thread_id; + + /// The state used in this object and inside the thread job. + Poco::Event event; + }; + std::shared_ptr state; + + /// Internally initialized() should be used over joinable(), + /// since it is enough to know that the thread is initialized, + /// and ignore that fact that thread cannot join itself. + bool initialized() const + { + return static_cast(state); + } }; diff --git a/src/Common/Throttler.h b/src/Common/Throttler.h index 89a83bb23be..6d44ad6ca5f 100644 --- a/src/Common/Throttler.h +++ b/src/Common/Throttler.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include @@ -57,7 +59,4 @@ private: std::shared_ptr parent; }; - -using ThrottlerPtr = std::shared_ptr; - } diff --git a/src/Common/Throttler_fwd.h b/src/Common/Throttler_fwd.h new file mode 100644 index 00000000000..1efaf1c85c5 --- /dev/null +++ b/src/Common/Throttler_fwd.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +namespace DB +{ + +class Throttler; +using ThrottlerPtr = std::shared_ptr; + +} diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index b917a0a1d13..00764269269 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -229,6 +229,35 @@ size_t getSizeFromFileDescriptor(int fd, const String & file_name) return buf.st_size; } +int getINodeNumberFromPath(const String & path) +{ + struct stat file_stat; + if (stat(path.data(), &file_stat)) + { + throwFromErrnoWithPath( + "Cannot execute stat for file " + path, + path, + ErrorCodes::CANNOT_STAT); + } + return file_stat.st_ino; +} + +std::optional tryGetSizeFromFilePath(const String & path) +{ + std::error_code ec; + + size_t size = fs::file_size(path, ec); + if (!ec) + return size; + + if (ec == std::errc::no_such_file_or_directory) + return std::nullopt; + if (ec == std::errc::operation_not_supported) + return std::nullopt; + + throw fs::filesystem_error("Got unexpected error while getting file size", path, ec); +} + } diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h index 1b8aa9bcdf9..2be3ea748fa 100644 --- a/src/Common/filesystemHelpers.h +++ b/src/Common/filesystemHelpers.h @@ -66,6 +66,12 @@ bool fileOrSymlinkPathStartsWith(const String & path, const String & prefix_path size_t getSizeFromFileDescriptor(int fd, const String & file_name = ""); +std::optional tryGetSizeFromFilePath(const String & path); + +/// Get inode number for a file path. +/// Will not work correctly if filesystem does not support inodes. +int getINodeNumberFromPath(const String & path); + } namespace FS diff --git a/src/Common/mysqlxx/mysqlxx/Exception.h b/src/Common/mysqlxx/mysqlxx/Exception.h index f022335a9dd..7886368e747 100644 --- a/src/Common/mysqlxx/mysqlxx/Exception.h +++ b/src/Common/mysqlxx/mysqlxx/Exception.h @@ -12,8 +12,8 @@ struct Exception : public Poco::Exception { explicit Exception(const std::string & msg, int code = 0) : Poco::Exception(msg, code) {} int errnum() const { return code(); } - const char * name() const throw() override { return "mysqlxx::Exception"; } - const char * className() const throw() override { return "mysqlxx::Exception"; } + const char * name() const noexcept override { return "mysqlxx::Exception"; } + const char * className() const noexcept override { return "mysqlxx::Exception"; } }; @@ -21,8 +21,8 @@ struct Exception : public Poco::Exception struct ConnectionFailed : public Exception { explicit ConnectionFailed(const std::string & msg, int code = 0) : Exception(msg, code) {} - const char * name() const throw() override { return "mysqlxx::ConnectionFailed"; } - const char * className() const throw() override { return "mysqlxx::ConnectionFailed"; } + const char * name() const noexcept override { return "mysqlxx::ConnectionFailed"; } + const char * className() const noexcept override { return "mysqlxx::ConnectionFailed"; } }; @@ -30,8 +30,8 @@ struct ConnectionFailed : public Exception struct ConnectionLost : public Exception { explicit ConnectionLost(const std::string & msg, int code = 0) : Exception(msg, code) {} - const char * name() const throw() override { return "mysqlxx::ConnectionLost"; } - const char * className() const throw() override { return "mysqlxx::ConnectionLost"; } + const char * name() const noexcept override { return "mysqlxx::ConnectionLost"; } + const char * className() const noexcept override { return "mysqlxx::ConnectionLost"; } }; @@ -39,8 +39,8 @@ struct ConnectionLost : public Exception struct BadQuery : public Exception { explicit BadQuery(const std::string & msg, int code = 0) : Exception(msg, code) {} - const char * name() const throw() override { return "mysqlxx::BadQuery"; } - const char * className() const throw() override { return "mysqlxx::BadQuery"; } + const char * name() const noexcept override { return "mysqlxx::BadQuery"; } + const char * className() const noexcept override { return "mysqlxx::BadQuery"; } }; @@ -48,8 +48,8 @@ struct BadQuery : public Exception struct CannotParseValue : public Exception { explicit CannotParseValue(const std::string & msg, int code = 0) : Exception(msg, code) {} - const char * name() const throw() override { return "mysqlxx::CannotParseValue"; } - const char * className() const throw() override { return "mysqlxx::CannotParseValue"; } + const char * name() const noexcept override { return "mysqlxx::CannotParseValue"; } + const char * className() const noexcept override { return "mysqlxx::CannotParseValue"; } }; diff --git a/src/Common/parseGlobs.cpp b/src/Common/parseGlobs.cpp index 75539512b6d..8e9195f9842 100644 --- a/src/Common/parseGlobs.cpp +++ b/src/Common/parseGlobs.cpp @@ -32,7 +32,7 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob } std::string escaped_with_globs = buf_for_escaping.str(); - static const re2::RE2 enum_or_range(R"({([\d]+\.\.[\d]+|[^{}*,]+,[^{}*]*[^{}*,])})"); /// regexp for {expr1,expr2,expr3} or {M..N}, where M and N - non-negative integers, expr's should be without {}*, + static const re2::RE2 enum_or_range(R"({([\d]+\.\.[\d]+|[^{}*,]+,[^{}*]*[^{}*,])})"); /// regexp for {expr1,expr2,expr3} or {M..N}, where M and N - non-negative integers, expr's should be without "{", "}", "*" and "," re2::StringPiece input(escaped_with_globs); re2::StringPiece matched; std::ostringstream oss_for_replacing; // STYLE_CHECK_ALLOW_STD_STRING_STREAM @@ -50,16 +50,32 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob char point; ReadBufferFromString buf_range(buffer); buf_range >> range_begin >> point >> point >> range_end; + + size_t range_begin_width = buffer.find('.'); + size_t range_end_width = buffer.size() - buffer.find_last_of('.') - 1; bool leading_zeros = buffer[0] == '0'; - size_t num_len = std::to_string(range_end).size(); + size_t output_width = 0; + + if (range_begin > range_end) //Descending Sequence {20..15} {9..01} + { + std::swap(range_begin,range_end); + leading_zeros = buffer[buffer.find_last_of('.')+1]=='0'; + std::swap(range_begin_width,range_end_width); + } + if (range_begin_width == 1 && leading_zeros) + output_width = 1; ///Special Case: {0..10} {0..999} + else + output_width = std::max(range_begin_width, range_end_width); + if (leading_zeros) - oss_for_replacing << std::setfill('0') << std::setw(num_len); + oss_for_replacing << std::setfill('0') << std::setw(output_width); oss_for_replacing << range_begin; + for (size_t i = range_begin + 1; i <= range_end; ++i) { oss_for_replacing << '|'; if (leading_zeros) - oss_for_replacing << std::setfill('0') << std::setw(num_len); + oss_for_replacing << std::setfill('0') << std::setw(output_width); oss_for_replacing << i; } } diff --git a/src/Common/tests/gtest_makeRegexpPatternFromGlobs.cpp b/src/Common/tests/gtest_makeRegexpPatternFromGlobs.cpp index 42777d0bbba..fda3a6ee1c8 100644 --- a/src/Common/tests/gtest_makeRegexpPatternFromGlobs.cpp +++ b/src/Common/tests/gtest_makeRegexpPatternFromGlobs.cpp @@ -8,14 +8,36 @@ using namespace DB; TEST(Common, makeRegexpPatternFromGlobs) { + EXPECT_EQ(makeRegexpPatternFromGlobs("?"), "[^/]"); EXPECT_EQ(makeRegexpPatternFromGlobs("*"), "[^/]*"); EXPECT_EQ(makeRegexpPatternFromGlobs("/?"), "/[^/]"); EXPECT_EQ(makeRegexpPatternFromGlobs("/*"), "/[^/]*"); EXPECT_EQ(makeRegexpPatternFromGlobs("*_{{a,b,c,d}}/?.csv"), "[^/]*_\\{(a|b|c|d)\\}/[^/]\\.csv"); - EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..09}"), "f(1|2|3|4|5|6|7|8|9)"); - EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..9}"), "f(1|2|3|4|5|6|7|8|9)"); - EXPECT_EQ(makeRegexpPatternFromGlobs("f{0001..0000009}"), "f(1|2|3|4|5|6|7|8|9)"); + /* Regex Parsing for {..} can have three possible cases + 1) The left range width == the right range width + 2) The left range width > the right range width + 3) The left range width < the right range width + */ + // Ascending Sequences + EXPECT_EQ(makeRegexpPatternFromGlobs("f{1..9}"), "f(1|2|3|4|5|6|7|8|9)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{0..10}"), "f(0|1|2|3|4|5|6|7|8|9|10)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{10..20}"), "f(10|11|12|13|14|15|16|17|18|19|20)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{00..10}"), "f(00|01|02|03|04|05|06|07|08|09|10)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{0001..0009}"), "f(0001|0002|0003|0004|0005|0006|0007|0008|0009)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..9}"), "f(01|02|03|04|05|06|07|08|09)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{000..9}"), "f(000|001|002|003|004|005|006|007|008|009)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{95..103}"), "f(95|96|97|98|99|100|101|102|103)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{99..109}"), "f(99|100|101|102|103|104|105|106|107|108|109)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{001..0009}"), "f(0001|0002|0003|0004|0005|0006|0007|0008|0009)"); + // Descending Sequences + EXPECT_EQ(makeRegexpPatternFromGlobs("f{20..15}"), "f(15|16|17|18|19|20)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{200..199}"), "f(199|200)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{0009..0001}"), "f(0001|0002|0003|0004|0005|0006|0007|0008|0009)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{100..90}"), "f(90|91|92|93|94|95|96|97|98|99|100)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{103..95}"), "f(95|96|97|98|99|100|101|102|103)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{9..01}"), "f(01|02|03|04|05|06|07|08|09)"); + EXPECT_EQ(makeRegexpPatternFromGlobs("f{9..000}"), "f(000|001|002|003|004|005|006|007|008|009)"); EXPECT_EQ(makeRegexpPatternFromGlobs("f{1..2}{1..2}"), "f(1|2)(1|2)"); EXPECT_EQ(makeRegexpPatternFromGlobs("f{1..1}{1..1}"), "f(1)(1)"); EXPECT_EQ(makeRegexpPatternFromGlobs("f{0..0}{0..0}"), "f(0)(0)"); diff --git a/src/Compression/CompressionCodecEncrypted.cpp b/src/Compression/CompressionCodecEncrypted.cpp index f7e597a0519..bf36fa114fb 100644 --- a/src/Compression/CompressionCodecEncrypted.cpp +++ b/src/Compression/CompressionCodecEncrypted.cpp @@ -131,7 +131,7 @@ std::string lastErrorString() /// This function get key and nonce and encrypt text with their help. /// If something went wrong (can't init context or can't encrypt data) it throws exception. /// It returns length of encrypted text. -size_t encrypt(const std::string_view & plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce) +size_t encrypt(std::string_view plaintext, char * ciphertext_and_tag, EncryptionMethod method, const String & key, const String & nonce) { /// Init context for encryption, using key. EVP_AEAD_CTX encrypt_ctx; @@ -160,7 +160,7 @@ size_t encrypt(const std::string_view & plaintext, char * ciphertext_and_tag, En /// This function get key and nonce and encrypt text with their help. /// If something went wrong (can't init context or can't encrypt data) it throws exception. /// It returns length of encrypted text. -size_t decrypt(const std::string_view & ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce) +size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod method, const String & key, const String & nonce) { /// Init context for decryption with given key. EVP_AEAD_CTX decrypt_ctx; diff --git a/src/Core/BaseSettings.cpp b/src/Core/BaseSettings.cpp index d4b2d1551b6..f03a59c1342 100644 --- a/src/Core/BaseSettings.cpp +++ b/src/Core/BaseSettings.cpp @@ -11,7 +11,7 @@ namespace ErrorCodes extern const int UNKNOWN_SETTING; } -void BaseSettingsHelpers::writeString(const std::string_view & str, WriteBuffer & out) +void BaseSettingsHelpers::writeString(std::string_view str, WriteBuffer & out) { writeStringBinary(str, out); } @@ -39,13 +39,13 @@ BaseSettingsHelpers::Flags BaseSettingsHelpers::readFlags(ReadBuffer & in) } -void BaseSettingsHelpers::throwSettingNotFound(const std::string_view & name) +void BaseSettingsHelpers::throwSettingNotFound(std::string_view name) { throw Exception("Unknown setting " + String{name}, ErrorCodes::UNKNOWN_SETTING); } -void BaseSettingsHelpers::warningSettingNotFound(const std::string_view & name) +void BaseSettingsHelpers::warningSettingNotFound(std::string_view name) { static auto * log = &Poco::Logger::get("Settings"); LOG_WARNING(log, "Unknown setting {}, skipping", name); diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index a4ddc6571ed..3638a036098 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -45,16 +45,16 @@ class BaseSettings : public TTraits::Data public: using Traits = TTraits; - void set(const std::string_view & name, const Field & value); - Field get(const std::string_view & name) const; + void set(std::string_view name, const Field & value); + Field get(std::string_view name) const; - void setString(const std::string_view & name, const String & value); - String getString(const std::string_view & name) const; + void setString(std::string_view name, const String & value); + String getString(std::string_view name) const; - bool tryGet(const std::string_view & name, Field & value) const; - bool tryGetString(const std::string_view & name, String & value) const; + bool tryGet(std::string_view name, Field & value) const; + bool tryGetString(std::string_view name, String & value) const; - bool isChanged(const std::string_view & name) const; + bool isChanged(std::string_view name) const; SettingsChanges changes() const; void applyChange(const SettingChange & change); void applyChanges(const SettingsChanges & changes); @@ -63,22 +63,22 @@ public: /// Resets all the settings to their default values. void resetToDefault(); - bool has(const std::string_view & name) const { return hasBuiltin(name) || hasCustom(name); } - static bool hasBuiltin(const std::string_view & name); - bool hasCustom(const std::string_view & name) const; + bool has(std::string_view name) const { return hasBuiltin(name) || hasCustom(name); } + static bool hasBuiltin(std::string_view name); + bool hasCustom(std::string_view name) const; - const char * getTypeName(const std::string_view & name) const; - const char * getDescription(const std::string_view & name) const; + const char * getTypeName(std::string_view name) const; + const char * getDescription(std::string_view name) const; /// Checks if it's possible to assign a field to a specified value and throws an exception if not. /// This function doesn't change the fields, it performs check only. - static void checkCanSet(const std::string_view & name, const Field & value); - static void checkCanSetString(const std::string_view & name, const String & str); + static void checkCanSet(std::string_view name, const Field & value); + static void checkCanSetString(std::string_view name, const String & str); /// Conversions without changing the fields. - static Field castValueUtil(const std::string_view & name, const Field & value); - static String valueToStringUtil(const std::string_view & name, const Field & value); - static Field stringToValueUtil(const std::string_view & name, const String & str); + static Field castValueUtil(std::string_view name, const Field & value); + static String valueToStringUtil(std::string_view name, const Field & value); + static Field stringToValueUtil(std::string_view name, const String & str); void write(WriteBuffer & out, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT) const; void read(ReadBuffer & in, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT); @@ -164,19 +164,19 @@ public: Iterator end() const { return allChanged().end(); } private: - SettingFieldCustom & getCustomSetting(const std::string_view & name); - const SettingFieldCustom & getCustomSetting(const std::string_view & name) const; - const SettingFieldCustom * tryGetCustomSetting(const std::string_view & name) const; + SettingFieldCustom & getCustomSetting(std::string_view name); + const SettingFieldCustom & getCustomSetting(std::string_view name) const; + const SettingFieldCustom * tryGetCustomSetting(std::string_view name) const; std::conditional_t custom_settings_map; }; struct BaseSettingsHelpers { - [[noreturn]] static void throwSettingNotFound(const std::string_view & name); - static void warningSettingNotFound(const std::string_view & name); + [[noreturn]] static void throwSettingNotFound(std::string_view name); + static void warningSettingNotFound(std::string_view name); - static void writeString(const std::string_view & str, WriteBuffer & out); + static void writeString(std::string_view str, WriteBuffer & out); static String readString(ReadBuffer & in); enum Flags : UInt64 @@ -190,7 +190,7 @@ struct BaseSettingsHelpers }; template -void BaseSettings::set(const std::string_view & name, const Field & value) +void BaseSettings::set(std::string_view name, const Field & value) { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -200,7 +200,7 @@ void BaseSettings::set(const std::string_view & name, const Field & val } template -Field BaseSettings::get(const std::string_view & name) const +Field BaseSettings::get(std::string_view name) const { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -210,7 +210,7 @@ Field BaseSettings::get(const std::string_view & name) const } template -void BaseSettings::setString(const std::string_view & name, const String & value) +void BaseSettings::setString(std::string_view name, const String & value) { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -220,7 +220,7 @@ void BaseSettings::setString(const std::string_view & name, const Strin } template -String BaseSettings::getString(const std::string_view & name) const +String BaseSettings::getString(std::string_view name) const { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -230,7 +230,7 @@ String BaseSettings::getString(const std::string_view & name) const } template -bool BaseSettings::tryGet(const std::string_view & name, Field & value) const +bool BaseSettings::tryGet(std::string_view name, Field & value) const { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -247,7 +247,7 @@ bool BaseSettings::tryGet(const std::string_view & name, Field & value) } template -bool BaseSettings::tryGetString(const std::string_view & name, String & value) const +bool BaseSettings::tryGetString(std::string_view name, String & value) const { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -264,7 +264,7 @@ bool BaseSettings::tryGetString(const std::string_view & name, String & } template -bool BaseSettings::isChanged(const std::string_view & name) const +bool BaseSettings::isChanged(std::string_view name) const { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -316,20 +316,20 @@ void BaseSettings::resetToDefault() } template -bool BaseSettings::hasBuiltin(const std::string_view & name) +bool BaseSettings::hasBuiltin(std::string_view name) { const auto & accessor = Traits::Accessor::instance(); return (accessor.find(name) != static_cast(-1)); } template -bool BaseSettings::hasCustom(const std::string_view & name) const +bool BaseSettings::hasCustom(std::string_view name) const { return tryGetCustomSetting(name); } template -const char * BaseSettings::getTypeName(const std::string_view & name) const +const char * BaseSettings::getTypeName(std::string_view name) const { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -341,7 +341,7 @@ const char * BaseSettings::getTypeName(const std::string_view & name) c } template -const char * BaseSettings::getDescription(const std::string_view & name) const +const char * BaseSettings::getDescription(std::string_view name) const { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -353,19 +353,19 @@ const char * BaseSettings::getDescription(const std::string_view & name } template -void BaseSettings::checkCanSet(const std::string_view & name, const Field & value) +void BaseSettings::checkCanSet(std::string_view name, const Field & value) { castValueUtil(name, value); } template -void BaseSettings::checkCanSetString(const std::string_view & name, const String & str) +void BaseSettings::checkCanSetString(std::string_view name, const String & str) { stringToValueUtil(name, str); } template -Field BaseSettings::castValueUtil(const std::string_view & name, const Field & value) +Field BaseSettings::castValueUtil(std::string_view name, const Field & value) { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -377,7 +377,7 @@ Field BaseSettings::castValueUtil(const std::string_view & name, const } template -String BaseSettings::valueToStringUtil(const std::string_view & name, const Field & value) +String BaseSettings::valueToStringUtil(std::string_view name, const Field & value) { const auto & accessor = Traits::Accessor::instance(); if (size_t index = accessor.find(name); index != static_cast(-1)) @@ -389,7 +389,7 @@ String BaseSettings::valueToStringUtil(const std::string_view & name, c } template -Field BaseSettings::stringToValueUtil(const std::string_view & name, const String & str) +Field BaseSettings::stringToValueUtil(std::string_view name, const String & str) { try { @@ -521,7 +521,7 @@ bool operator!=(const BaseSettings & left, const BaseSettings } template -SettingFieldCustom & BaseSettings::getCustomSetting(const std::string_view & name) +SettingFieldCustom & BaseSettings::getCustomSetting(std::string_view name) { if constexpr (Traits::allow_custom_settings) { @@ -537,7 +537,7 @@ SettingFieldCustom & BaseSettings::getCustomSetting(const std::string_v } template -const SettingFieldCustom & BaseSettings::getCustomSetting(const std::string_view & name) const +const SettingFieldCustom & BaseSettings::getCustomSetting(std::string_view name) const { if constexpr (Traits::allow_custom_settings) { @@ -549,7 +549,7 @@ const SettingFieldCustom & BaseSettings::getCustomSetting(const std::st } template -const SettingFieldCustom * BaseSettings::tryGetCustomSetting(const std::string_view & name) const +const SettingFieldCustom * BaseSettings::tryGetCustomSetting(std::string_view name) const { if constexpr (Traits::allow_custom_settings) { @@ -780,7 +780,7 @@ bool BaseSettings::SettingFieldRef::isObsolete() const public: \ static const Accessor & instance(); \ size_t size() const { return field_infos.size(); } \ - size_t find(const std::string_view & name) const; \ + size_t find(std::string_view name) const; \ const String & getName(size_t index) const { return field_infos[index].name; } \ const char * getTypeName(size_t index) const { return field_infos[index].type; } \ const char * getDescription(size_t index) const { return field_infos[index].description; } \ @@ -851,7 +851,7 @@ bool BaseSettings::SettingFieldRef::isObsolete() const \ SETTINGS_TRAITS_NAME::Accessor::Accessor() {} \ \ - size_t SETTINGS_TRAITS_NAME::Accessor::find(const std::string_view & name) const \ + size_t SETTINGS_TRAITS_NAME::Accessor::find(std::string_view name) const \ { \ auto it = name_to_index_map.find(name); \ if (it != name_to_index_map.end()) \ diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 3a4b66e6266..acdfca7a7b2 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -286,7 +286,7 @@ String Field::dump() const return applyVisitor(FieldVisitorDump(), *this); } -Field Field::restoreFromDump(const std::string_view & dump_) +Field Field::restoreFromDump(std::string_view dump_) { auto show_error = [&dump_] { diff --git a/src/Core/Field.h b/src/Core/Field.h index 08274876914..f60b7e4902e 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -346,7 +346,7 @@ public: } /// Create a string inplace. - Field(const std::string_view & str) { create(str.data(), str.size()); } /// NOLINT + Field(std::string_view str) { create(str.data(), str.size()); } /// NOLINT Field(const String & str) { create(std::string_view{str}); } /// NOLINT Field(String && str) { create(std::move(str)); } /// NOLINT Field(const char * str) { create(std::string_view{str}); } /// NOLINT @@ -403,7 +403,7 @@ public: return *this; } - Field & operator= (const std::string_view & str); + Field & operator= (std::string_view str); Field & operator= (const String & str) { return *this = std::string_view{str}; } Field & operator= (String && str); Field & operator= (const char * str) { return *this = std::string_view{str}; } @@ -631,7 +631,7 @@ public: } String dump() const; - static Field restoreFromDump(const std::string_view & dump_); + static Field restoreFromDump(std::string_view dump_); private: std::aligned_union_t()) {} - SettingFieldString & operator =(const std::string_view & str) { value = str; changed = true; return *this; } + SettingFieldString & operator =(std::string_view str) { value = str; changed = true; return *this; } SettingFieldString & operator =(const String & str) { *this = std::string_view{str}; return *this; } SettingFieldString & operator =(String && str) { value = std::move(str); changed = true; return *this; } SettingFieldString & operator =(const char * str) { *this = std::string_view{str}; return *this; } @@ -256,7 +256,7 @@ struct SettingFieldEnum struct SettingFieldEnumHelpers { - static void writeBinary(const std::string_view & str, WriteBuffer & out); + static void writeBinary(std::string_view str, WriteBuffer & out); static String readBinary(ReadBuffer & in); }; @@ -286,7 +286,7 @@ void SettingFieldEnum::readBinary(ReadBuffer & in) { \ using EnumType = ENUM_TYPE; \ static const String & toString(EnumType value); \ - static EnumType fromString(const std::string_view & str); \ + static EnumType fromString(std::string_view str); \ }; \ \ using SettingField##NEW_NAME = SettingFieldEnum; @@ -310,7 +310,7 @@ void SettingFieldEnum::readBinary(ReadBuffer & in) ERROR_CODE_FOR_UNEXPECTED_NAME); \ } \ \ - typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(const std::string_view & str) \ + typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(std::string_view str) \ { \ static const std::unordered_map map = [] { \ std::unordered_map res; \ @@ -430,7 +430,7 @@ void SettingFieldMultiEnum::readBinary(ReadBuffer & in) using EnumType = ENUM_TYPE; \ static size_t getEnumSize(); \ static const String & toString(EnumType value); \ - static EnumType fromString(const std::string_view & str); \ + static EnumType fromString(std::string_view str); \ }; \ \ using SettingField##NEW_NAME = SettingFieldMultiEnum; diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index 5dae2b7b413..b28b70f676a 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -54,7 +54,7 @@ std::pair splitName(const std::string & name, bool rev return {name.substr(0, idx), name.substr(idx + 1)}; } -std::pair splitName(const std::string_view & name, bool reverse) +std::pair splitName(std::string_view name, bool reverse) { auto idx = (reverse ? name.find_last_of('.') : name.find_first_of('.')); if (idx == std::string::npos || idx == 0 || idx + 1 == name.size()) diff --git a/src/DataTypes/NestedUtils.h b/src/DataTypes/NestedUtils.h index e7cda541f47..38da382254c 100644 --- a/src/DataTypes/NestedUtils.h +++ b/src/DataTypes/NestedUtils.h @@ -13,7 +13,7 @@ namespace Nested /// Splits name of compound identifier by first/last dot (depending on 'reverse' parameter). std::pair splitName(const std::string & name, bool reverse = false); - std::pair splitName(const std::string_view & name, bool reverse = false); + std::pair splitName(std::string_view name, bool reverse = false); /// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot. std::string extractTableName(const std::string & nested_name); diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index 44248cae841..52e323487f2 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -161,6 +161,11 @@ void DiskDecorator::removeSharedFile(const String & path, bool keep_s3) delegate->removeSharedFile(path, keep_s3); } +void DiskDecorator::removeSharedFileIfExists(const String & path, bool keep_s3) +{ + delegate->removeSharedFileIfExists(path, keep_s3); +} + void DiskDecorator::removeSharedFiles(const RemoveBatchRequest & files, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) { delegate->removeSharedFiles(files, keep_all_batch_data, file_names_remove_metadata_only); @@ -231,4 +236,9 @@ void DiskDecorator::applyNewSettings(const Poco::Util::AbstractConfiguration & c delegate->applyNewSettings(config, context, config_prefix, map); } +DiskObjectStoragePtr DiskDecorator::createDiskObjectStorage(const String & name) +{ + return delegate->createDiskObjectStorage(name); +} + } diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index 99cbb240b98..1eac37e13f3 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -52,11 +52,15 @@ public: void removeFile(const String & path) override; void removeFileIfExists(const String & path) override; + void removeSharedFileIfExists(const String & path, bool keep_s3) override; + void removeDirectory(const String & path) override; void removeRecursive(const String & path) override; + void removeSharedFile(const String & path, bool keep_s3) override; void removeSharedRecursive(const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) override; void removeSharedFiles(const RemoveBatchRequest & files, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) override; + void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; time_t getLastChanged(const String & path) const override; Poco::Timestamp getLastModified(const String & path) const override; @@ -77,9 +81,14 @@ public: void shutdown() override; void startup(ContextPtr context) override; void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override; + + bool supportsCache() const override { return delegate->supportsCache(); } String getCacheBasePath() const override { return delegate->getCacheBasePath(); } - PathsWithSize getObjectStoragePaths(const String & path) const override { return delegate->getObjectStoragePaths(path); } - void getRemotePathsRecursive(const String & path, std::vector & paths_map) override { return delegate->getRemotePathsRecursive(path, paths_map); } + + StoredObjects getStorageObjects(const String & path) const override { return delegate->getStorageObjects(path); } + DiskObjectStoragePtr createDiskObjectStorage(const String &) override; + + void getRemotePathsRecursive(const String & path, std::vector & paths_map) override { return delegate->getRemotePathsRecursive(path, paths_map); } MetadataStoragePtr getMetadataStorage() override { return delegate->getMetadataStorage(); } diff --git a/src/Disks/DiskRestartProxy.cpp b/src/Disks/DiskRestartProxy.cpp index 99dfc8e96a0..01c9b176853 100644 --- a/src/Disks/DiskRestartProxy.cpp +++ b/src/Disks/DiskRestartProxy.cpp @@ -318,13 +318,14 @@ String DiskRestartProxy::getCacheBasePath() const return DiskDecorator::getCacheBasePath(); } -PathsWithSize DiskRestartProxy::getObjectStoragePaths(const String & path) const +StoredObjects DiskRestartProxy::getStorageObjects(const String & path) const { ReadLock lock (mutex); - return DiskDecorator::getObjectStoragePaths(path); + return DiskDecorator::getStorageObjects(path); } -void DiskRestartProxy::getRemotePathsRecursive(const String & path, std::vector & paths_map) +void DiskRestartProxy::getRemotePathsRecursive( + const String & path, std::vector & paths_map) { ReadLock lock (mutex); return DiskDecorator::getRemotePathsRecursive(path, paths_map); diff --git a/src/Disks/DiskRestartProxy.h b/src/Disks/DiskRestartProxy.h index e483936c817..07c2b808a04 100644 --- a/src/Disks/DiskRestartProxy.h +++ b/src/Disks/DiskRestartProxy.h @@ -65,8 +65,8 @@ public: String getUniqueId(const String & path) const override; bool checkUniqueId(const String & id) const override; String getCacheBasePath() const override; - PathsWithSize getObjectStoragePaths(const String & path) const override; - void getRemotePathsRecursive(const String & path, std::vector & paths_map) override; + StoredObjects getStorageObjects(const String & path) const override; + void getRemotePathsRecursive(const String & path, std::vector & paths_map) override; void restart(ContextPtr context); diff --git a/src/Disks/DiskWebServer.cpp b/src/Disks/DiskWebServer.cpp index 83cfbbb0150..54dce926893 100644 --- a/src/Disks/DiskWebServer.cpp +++ b/src/Disks/DiskWebServer.cpp @@ -170,10 +170,10 @@ std::unique_ptr DiskWebServer::readFile(const String & p auto remote_path = fs_path.parent_path() / (escapeForFileName(fs_path.stem()) + fs_path.extension().string()); remote_path = remote_path.string().substr(url.size()); - PathsWithSize blobs_to_read; - blobs_to_read.emplace_back(remote_path, iter->second.size); + StoredObjects objects; + objects.emplace_back(remote_path, iter->second.size); - auto web_impl = std::make_unique(url, blobs_to_read, getContext(), read_settings); + auto web_impl = std::make_unique(url, objects, getContext(), read_settings); if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { diff --git a/src/Disks/DiskWebServer.h b/src/Disks/DiskWebServer.h index 40754e71fa0..4dc476894f9 100644 --- a/src/Disks/DiskWebServer.h +++ b/src/Disks/DiskWebServer.h @@ -144,6 +144,11 @@ public: throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk {} is read-only", getName()); } + void removeSharedFileIfExists(const String &, bool) override + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk {} is read-only", getName()); + } + void removeSharedRecursive(const String &, bool, const NameSet &) override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk {} is read-only", getName()); @@ -169,9 +174,9 @@ public: throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk {} is read-only", getName()); } - PathsWithSize getObjectStoragePaths(const String &) const override { return {}; } + StoredObjects getStorageObjects(const String &) const override { return {}; } - void getRemotePathsRecursive(const String &, std::vector &) override {} + void getRemotePathsRecursive(const String &, std::vector &) override {} /// Create part diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index 503e926743a..7d89fb28271 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include namespace DB @@ -119,6 +121,18 @@ SyncGuardPtr IDisk::getDirectorySyncGuard(const String & /* path */) const return nullptr; } -MetadataStoragePtr IDisk::getMetadataStorage() { return std::make_shared(std::static_pointer_cast(shared_from_this()), ""); } +MetadataStoragePtr IDisk::getMetadataStorage() +{ + if (isRemote()) + { + return std::make_shared(std::static_pointer_cast(shared_from_this()), ""); + } + else + { + auto object_storage = std::make_shared(); + return std::make_shared( + std::static_pointer_cast(shared_from_this()), object_storage, getPath()); + } +} } diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 27cac7a5456..941df99298b 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -55,6 +55,8 @@ using DiskTransactionPtr = std::shared_ptr; struct RemoveRequest; using RemoveBatchRequest = std::vector; +class DiskObjectStorage; +using DiskObjectStoragePtr = std::shared_ptr; /** * Provide interface for reservation. @@ -215,19 +217,22 @@ public: /// Second bool param is a flag to remove (true) or keep (false) shared data on S3 virtual void removeSharedFileIfExists(const String & path, bool /* keep_shared_data */) { removeFileIfExists(path); } - virtual String getCacheBasePath() const { return ""; } + virtual String getCacheBasePath() const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no cache path"); } - /// Returns a list of paths because for Log family engines there might be - /// multiple files in remote fs for single clickhouse file. - virtual PathsWithSize getObjectStoragePaths(const String &) const + virtual bool supportsCache() const { return false; } + + /// Returns a list of storage objects (contains path, size, ...). + /// (A list is returned because for Log family engines there might + /// be multiple files in remote fs for single clickhouse file. + virtual StoredObjects getStorageObjects(const String &) const { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getObjectStoragePaths() not implemented for disk: {}`", getType()); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getStorageObjects() not implemented for disk: {}`", getType()); } /// For one local path there might be multiple remote paths in case of Log family engines. - using LocalPathWithRemotePaths = std::pair; + using LocalPathWithObjectStoragePaths = std::pair; - virtual void getRemotePathsRecursive(const String &, std::vector &) + virtual void getRemotePathsRecursive(const String &, std::vector &) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getRemotePathsRecursive() not implemented for disk: {}`", getType()); } @@ -335,6 +340,13 @@ public: /// Return current disk revision. virtual UInt64 getRevision() const { return 0; } + virtual DiskObjectStoragePtr createDiskObjectStorage(const String &) + { + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Method createDiskObjectStorage() is not implemented for disk type: {}", + getType()); + } protected: friend class DiskDecorator; diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h index 4b00a9bcefc..74fbe8919fe 100644 --- a/src/Disks/IDiskTransaction.h +++ b/src/Disks/IDiskTransaction.h @@ -10,7 +10,7 @@ namespace DB struct RemoveRequest { - std::string path; + std::string path; /// Relative path. bool if_exists = false; explicit RemoveRequest(std::string path_, bool if_exists_ = false) diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp index 46f18d32bf1..b929cea0236 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -567,28 +567,31 @@ void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment) ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceBytes, current_impl_buffer_size); - if (file_segment->reserve(current_predownload_size)) + bool continue_predownload = file_segment->reserve(current_predownload_size); + if (continue_predownload) { LOG_TEST(log, "Left to predownload: {}, buffer size: {}", bytes_to_predownload, current_impl_buffer_size); assert(file_segment->getDownloadOffset() == static_cast(implementation_buffer->getPosition())); - Stopwatch watch(CLOCK_MONOTONIC); + bool success = writeCache(implementation_buffer->buffer().begin(), current_predownload_size, current_offset, *file_segment); + if (success) + { + current_offset += current_predownload_size; - file_segment->write(implementation_buffer->buffer().begin(), current_predownload_size, current_offset); + bytes_to_predownload -= current_predownload_size; + implementation_buffer->position() += current_predownload_size; + } + else + { + read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; + file_segment->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); - watch.stop(); - auto elapsed = watch.elapsedMicroseconds(); - current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed); - ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed); - ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, current_predownload_size); - - current_offset += current_predownload_size; - - bytes_to_predownload -= current_predownload_size; - implementation_buffer->position() += current_predownload_size; + continue_predownload = false; + } } - else + + if (!continue_predownload) { /// We were predownloading: /// segment{1} @@ -691,6 +694,34 @@ bool CachedReadBufferFromRemoteFS::updateImplementationBufferIfNeeded() return true; } +bool CachedReadBufferFromRemoteFS::writeCache(char * data, size_t size, size_t offset, FileSegment & file_segment) +{ + Stopwatch watch(CLOCK_MONOTONIC); + + try + { + file_segment.write(data, size, offset); + } + catch (ErrnoException & e) + { + int code = e.getErrno(); + if (code == /* No space left on device */28 || code == /* Quota exceeded */122) + { + LOG_INFO(log, "Insert into cache is skipped due to insufficient disk space. ({})", e.displayText()); + return false; + } + throw; + } + + watch.stop(); + auto elapsed = watch.elapsedMicroseconds(); + current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed); + ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed); + ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, size); + + return true; +} + bool CachedReadBufferFromRemoteFS::nextImpl() { try @@ -840,33 +871,34 @@ bool CachedReadBufferFromRemoteFS::nextImplStep() { assert(file_offset_of_buffer_end + size - 1 <= file_segment->range().right); - if (file_segment->reserve(size)) + bool success = file_segment->reserve(size); + if (success) { assert(file_segment->getDownloadOffset() == static_cast(implementation_buffer->getPosition())); - Stopwatch watch(CLOCK_MONOTONIC); - - file_segment->write( - needed_to_predownload ? implementation_buffer->position() : implementation_buffer->buffer().begin(), - size, - file_offset_of_buffer_end); - - watch.stop(); - auto elapsed = watch.elapsedMicroseconds(); - current_file_segment_counters.increment(ProfileEvents::FileSegmentCacheWriteMicroseconds, elapsed); - ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteMicroseconds, elapsed); - ProfileEvents::increment(ProfileEvents::CachedReadBufferCacheWriteBytes, size); - - assert(file_segment->getDownloadOffset() <= file_segment->range().right + 1); - assert( - std::next(current_file_segment_it) == file_segments_holder->file_segments.end() - || file_segment->getDownloadOffset() == implementation_buffer->getFileOffsetOfBufferEnd()); + success = writeCache(implementation_buffer->position(), size, file_offset_of_buffer_end, *file_segment); + if (success) + { + assert(file_segment->getDownloadOffset() <= file_segment->range().right + 1); + assert( + std::next(current_file_segment_it) == file_segments_holder->file_segments.end() + || file_segment->getDownloadOffset() == implementation_buffer->getFileOffsetOfBufferEnd()); + } + else + { + assert(file_segment->state() == FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); + } } else { - download_current_segment = false; - file_segment->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); LOG_DEBUG(log, "No space left in cache, will continue without cache download"); + file_segment->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); + } + + if (!success) + { + read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; + download_current_segment = false; } } diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.h b/src/Disks/IO/CachedReadBufferFromRemoteFS.h index 867b8538260..aff29dd200c 100644 --- a/src/Disks/IO/CachedReadBufferFromRemoteFS.h +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.h @@ -73,10 +73,13 @@ private: SeekableReadBufferPtr getRemoteFSReadBuffer(FileSegmentPtr & file_segment, ReadType read_type_); size_t getTotalSizeToRead(); + bool completeFileSegmentAndGetNext(); void appendFilesystemCacheLog(const FileSegment::Range & file_segment_range, ReadType read_type); + bool writeCache(char * data, size_t size, size_t offset, FileSegment & file_segment); + Poco::Logger * log; IFileCache::Key cache_key; String remote_fs_object_path; diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 1a4ae3f963f..14614871185 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -87,9 +87,9 @@ SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementation return std::make_unique( blob_container_client, path, + settings, max_single_read_retries, max_single_download_retries, - settings.remote_fs_buffer_size, /* use_external_buffer */true, read_until_position); } @@ -115,13 +115,13 @@ SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBufferImpl(c auto hdfs_uri = path.substr(0, begin_of_path); LOG_TEST(log, "HDFS uri: {}, path: {}", hdfs_path, hdfs_uri); - return std::make_unique(hdfs_uri, hdfs_path, config); + return std::make_unique(hdfs_uri, hdfs_path, config, settings); } #endif ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather( - const PathsWithSize & blobs_to_read_, + const StoredObjects & blobs_to_read_, const ReadSettings & settings_) : ReadBuffer(nullptr, 0) , blobs_to_read(blobs_to_read_) @@ -130,6 +130,9 @@ ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather( , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather")) , enable_cache_log(!query_id.empty() && settings.enable_filesystem_cache_log) { + if (blobs_to_read.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read zero number of objects"); + with_cache = settings.remote_fs_cache && settings.enable_filesystem_cache && (!IFileCache::isReadOnly() || settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache); @@ -182,22 +185,22 @@ void ReadBufferFromRemoteFSGather::initialize() auto current_buf_offset = file_offset_of_buffer_end; for (size_t i = 0; i < blobs_to_read.size(); ++i) { - const auto & [file_path, size] = blobs_to_read[i]; + const auto & object = blobs_to_read[i]; - if (size > current_buf_offset) + if (object.bytes_size > current_buf_offset) { /// Do not create a new buffer if we already have what we need. if (!current_buf || current_buf_idx != i) { current_buf_idx = i; - current_buf = createImplementationBuffer(file_path, size); + current_buf = createImplementationBuffer(object.absolute_path, object.bytes_size); } current_buf->seek(current_buf_offset, SEEK_SET); return; } - current_buf_offset -= size; + current_buf_offset -= object.bytes_size; } current_buf_idx = blobs_to_read.size(); current_buf = nullptr; @@ -217,10 +220,14 @@ bool ReadBufferFromRemoteFSGather::nextImpl() return true; } else + { return false; + } if (!moveToNextBuffer()) + { return false; + } return readImpl(); } @@ -234,8 +241,8 @@ bool ReadBufferFromRemoteFSGather::moveToNextBuffer() ++current_buf_idx; - const auto & [path, size] = blobs_to_read[current_buf_idx]; - current_buf = createImplementationBuffer(path, size); + const auto & object = blobs_to_read[current_buf_idx]; + current_buf = createImplementationBuffer(object.absolute_path, object.bytes_size); return true; } diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index 6a39f4add46..e065c714e04 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -27,7 +27,7 @@ friend class ReadIndirectBufferFromRemoteFS; public: ReadBufferFromRemoteFSGather( - const PathsWithSize & blobs_to_read_, + const StoredObjects & blobs_to_read_, const ReadSettings & settings_); ~ReadBufferFromRemoteFSGather() override; @@ -53,7 +53,7 @@ public: protected: virtual SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) = 0; - PathsWithSize blobs_to_read; + StoredObjects blobs_to_read; ReadSettings settings; @@ -109,7 +109,7 @@ public: std::shared_ptr client_ptr_, const String & bucket_, const String & version_id_, - const PathsWithSize & blobs_to_read_, + const StoredObjects & blobs_to_read_, size_t max_single_read_retries_, const ReadSettings & settings_) : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) @@ -138,7 +138,7 @@ class ReadBufferFromAzureBlobStorageGather final : public ReadBufferFromRemoteFS public: ReadBufferFromAzureBlobStorageGather( std::shared_ptr blob_container_client_, - const PathsWithSize & blobs_to_read_, + const StoredObjects & blobs_to_read_, size_t max_single_read_retries_, size_t max_single_download_retries_, const ReadSettings & settings_) @@ -164,7 +164,7 @@ class ReadBufferFromWebServerGather final : public ReadBufferFromRemoteFSGather public: ReadBufferFromWebServerGather( const String & uri_, - const PathsWithSize & blobs_to_read_, + const StoredObjects & blobs_to_read_, ContextPtr context_, const ReadSettings & settings_) : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) @@ -188,7 +188,7 @@ class ReadBufferFromHDFSGather final : public ReadBufferFromRemoteFSGather public: ReadBufferFromHDFSGather( const Poco::Util::AbstractConfiguration & config_, - const PathsWithSize & blobs_to_read_, + const StoredObjects & blobs_to_read_, const ReadSettings & settings_) : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) , config(config_) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index 3dcdd47826f..590433e91e5 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -5,6 +5,7 @@ #include #include #include +#include using namespace Azure::Storage::Blobs; diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index c25d1d7470c..37fbc0fb05c 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -2,6 +2,7 @@ #if USE_AZURE_BLOB_STORAGE +#include #include #include #include @@ -9,6 +10,7 @@ #include + namespace DB { @@ -21,24 +23,27 @@ namespace ErrorCodes AzureObjectStorage::AzureObjectStorage( - FileCachePtr && cache_, const String & name_, AzureClientPtr && client_, SettingsPtr && settings_) - : IObjectStorage(std::move(cache_)) - , name(name_) + : name(name_) , client(std::move(client_)) , settings(std::move(settings_)) { } -bool AzureObjectStorage::exists(const std::string & uri) const +std::string AzureObjectStorage::generateBlobNameForPath(const std::string & /* path */) +{ + return getRandomASCIIString(); +} + +bool AzureObjectStorage::exists(const StoredObject & object) const { auto client_ptr = client.get(); /// What a shame, no Exists method... Azure::Storage::Blobs::ListBlobsOptions options; - options.Prefix = uri; + options.Prefix = object.absolute_path; options.PageSizeHint = 1; auto blobs_list_response = client_ptr->ListBlobs(options); @@ -46,15 +51,15 @@ bool AzureObjectStorage::exists(const std::string & uri) const for (const auto & blob : blobs_list) { - if (uri == blob.Name) + if (object.absolute_path == blob.Name) return true; } return false; } -std::unique_ptr AzureObjectStorage::readObject( /// NOLINT - const std::string & path, +std::unique_ptr AzureObjectStorage::readObject( /// NOLINT + const StoredObject & object, const ReadSettings & read_settings, std::optional, std::optional) const @@ -62,28 +67,29 @@ std::unique_ptr AzureObjectStorage::readObject( /// NOLINT auto settings_ptr = settings.get(); return std::make_unique( - client.get(), path, settings_ptr->max_single_read_retries, - settings_ptr->max_single_download_retries, read_settings.remote_fs_buffer_size); + client.get(), object.absolute_path, patchSettings(read_settings), settings_ptr->max_single_read_retries, + settings_ptr->max_single_download_retries); } std::unique_ptr AzureObjectStorage::readObjects( /// NOLINT - const PathsWithSize & paths_to_read, + const StoredObjects & objects, const ReadSettings & read_settings, std::optional, std::optional) const { + ReadSettings disk_read_settings = patchSettings(read_settings); auto settings_ptr = settings.get(); auto reader_impl = std::make_unique( client.get(), - paths_to_read, + objects, settings_ptr->max_single_read_retries, settings_ptr->max_single_download_retries, - read_settings); + disk_read_settings); - if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) + if (disk_read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { auto reader = getThreadPoolReader(); - return std::make_unique(reader, read_settings, std::move(reader_impl)); + return std::make_unique(reader, disk_read_settings, std::move(reader_impl)); } else { @@ -94,23 +100,24 @@ std::unique_ptr AzureObjectStorage::readObjects( /// NOL /// Open the file for write and return WriteBufferFromFileBase object. std::unique_ptr AzureObjectStorage::writeObject( /// NOLINT - const std::string & path, + const StoredObject & object, WriteMode mode, std::optional, FinalizeCallback && finalize_callback, size_t buf_size, - const WriteSettings &) + const WriteSettings & write_settings) { if (mode != WriteMode::Rewrite) throw Exception("Azure storage doesn't support append", ErrorCodes::UNSUPPORTED_METHOD); auto buffer = std::make_unique( client.get(), - path, + object.absolute_path, settings.get()->max_single_part_upload_size, - buf_size); + buf_size, + patchSettings(write_settings)); - return std::make_unique(std::move(buffer), std::move(finalize_callback), path); + return std::make_unique(std::move(buffer), std::move(finalize_callback), object.absolute_path); } void AzureObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const @@ -128,36 +135,37 @@ void AzureObjectStorage::listPrefix(const std::string & path, RelativePathsWithS } /// Remove file. Throws exception if file doesn't exists or it's a directory. -void AzureObjectStorage::removeObject(const std::string & path) +void AzureObjectStorage::removeObject(const StoredObject & object) { + const auto & path = object.absolute_path; auto client_ptr = client.get(); auto delete_info = client_ptr->DeleteBlob(path); if (!delete_info.Value.Deleted) throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file in AzureBlob Storage: {}", path); } -void AzureObjectStorage::removeObjects(const PathsWithSize & paths) +void AzureObjectStorage::removeObjects(const StoredObjects & objects) { auto client_ptr = client.get(); - for (const auto & [path, _] : paths) + for (const auto & object : objects) { - auto delete_info = client_ptr->DeleteBlob(path); + auto delete_info = client_ptr->DeleteBlob(object.absolute_path); if (!delete_info.Value.Deleted) - throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file in AzureBlob Storage: {}", path); + throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file in AzureBlob Storage: {}", object.absolute_path); } } -void AzureObjectStorage::removeObjectIfExists(const std::string & path) +void AzureObjectStorage::removeObjectIfExists(const StoredObject & object) { auto client_ptr = client.get(); - auto delete_info = client_ptr->DeleteBlob(path); + auto delete_info = client_ptr->DeleteBlob(object.absolute_path); } -void AzureObjectStorage::removeObjectsIfExist(const PathsWithSize & paths) +void AzureObjectStorage::removeObjectsIfExist(const StoredObjects & objects) { auto client_ptr = client.get(); - for (const auto & [path, _] : paths) - auto delete_info = client_ptr->DeleteBlob(path); + for (const auto & object : objects) + auto delete_info = client_ptr->DeleteBlob(object.absolute_path); } @@ -179,13 +187,14 @@ ObjectMetadata AzureObjectStorage::getObjectMetadata(const std::string & path) c } void AzureObjectStorage::copyObject( /// NOLINT - const std::string & object_from, - const std::string & object_to, + const StoredObject & object_from, + const StoredObject & object_to, std::optional object_to_attributes) { auto client_ptr = client.get(); - auto dest_blob_client = client_ptr->GetBlobClient(object_to); - auto source_blob_client = client_ptr->GetBlobClient(object_from); + auto dest_blob_client = client_ptr->GetBlobClient(object_to.absolute_path); + auto source_blob_client = client_ptr->GetBlobClient(object_from.absolute_path); + Azure::Storage::Blobs::CopyBlobFromUriOptions copy_options; if (object_to_attributes.has_value()) { @@ -200,7 +209,7 @@ void AzureObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguratio { auto new_settings = getAzureBlobStorageSettings(config, config_prefix, context); settings.set(std::move(new_settings)); - + applyRemoteThrottlingSettings(context); /// We don't update client } @@ -208,7 +217,6 @@ void AzureObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguratio std::unique_ptr AzureObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) { return std::make_unique( - nullptr, name, getAzureBlobContainerClient(config, config_prefix), getAzureBlobStorageSettings(config, config_prefix, context) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index ab7d2b28508..6df093ebd43 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -45,28 +45,29 @@ public: using SettingsPtr = std::unique_ptr; AzureObjectStorage( - FileCachePtr && cache_, const String & name_, AzureClientPtr && client_, SettingsPtr && settings_); - bool exists(const std::string & uri) const override; + std::string getName() const override { return "AzureObjectStorage"; } - std::unique_ptr readObject( /// NOLINT - const std::string & path, + bool exists(const StoredObject & object) const override; + + std::unique_ptr readObject( /// NOLINT + const StoredObject & object, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT - const PathsWithSize & blobs_to_read, + const StoredObjects & objects, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; /// Open the file for write and return WriteBufferFromFileBase object. std::unique_ptr writeObject( /// NOLINT - const std::string & path, + const StoredObject & object, WriteMode mode, std::optional attributes = {}, FinalizeCallback && finalize_callback = {}, @@ -76,19 +77,19 @@ public: void listPrefix(const std::string & path, RelativePathsWithSize & children) const override; /// Remove file. Throws exception if file doesn't exists or it's a directory. - void removeObject(const std::string & path) override; + void removeObject(const StoredObject & object) override; - void removeObjects(const PathsWithSize & paths) override; + void removeObjects(const StoredObjects & objects) override; - void removeObjectIfExists(const std::string & path) override; + void removeObjectIfExists(const StoredObject & object) override; - void removeObjectsIfExist(const PathsWithSize & paths) override; + void removeObjectsIfExist(const StoredObjects & objects) override; ObjectMetadata getObjectMetadata(const std::string & path) const override; void copyObject( /// NOLINT - const std::string & object_from, - const std::string & object_to, + const StoredObject & object_from, + const StoredObject & object_to, std::optional object_to_attributes = {}) override; void shutdown() override {} @@ -108,6 +109,9 @@ public: const std::string & config_prefix, ContextPtr context) override; + std::string generateBlobNameForPath(const std::string & path) override; + + bool isRemote() const override { return true; } private: const String name; diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp index 53b484ae674..44976b7cf2d 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp @@ -5,7 +5,7 @@ #if USE_AZURE_BLOB_STORAGE #include -#include + #include #include @@ -73,9 +73,7 @@ void registerDiskAzureBlobStorage(DiskFactory & factory) { auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context); - /// FIXME Cache currently unsupported :( ObjectStoragePtr azure_object_storage = std::make_unique( - nullptr, name, getAzureBlobContainerClient(config, config_prefix), getAzureBlobStorageSettings(config, config_prefix, context)); @@ -104,6 +102,8 @@ void registerDiskAzureBlobStorage(DiskFactory & factory) checkRemoveAccess(*azure_blob_storage_disk); } + azure_blob_storage_disk->startup(context); + #ifdef NDEBUG bool use_cache = true; #else @@ -111,17 +111,14 @@ void registerDiskAzureBlobStorage(DiskFactory & factory) /// read buffer. bool use_cache = false; #endif - - azure_blob_storage_disk->startup(context); - if (config.getBool(config_prefix + ".cache_enabled", use_cache)) { String cache_path = config.getString(config_prefix + ".cache_path", context->getPath() + "disks/" + name + "/cache/"); azure_blob_storage_disk = wrapWithCache(azure_blob_storage_disk, "azure-blob-storage-cache", cache_path, metadata_path); } - return std::make_shared(azure_blob_storage_disk); }; + factory.registerDiskType("azure_blob_storage", creator); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index a354ad27049..ca414a7ee72 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -9,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -89,43 +89,43 @@ DiskTransactionPtr DiskObjectStorage::createObjectStorageTransaction() return std::make_shared( *object_storage, *metadata_storage, - remote_fs_root_path, send_metadata ? metadata_helper.get() : nullptr); } DiskObjectStorage::DiskObjectStorage( const String & name_, - const String & remote_fs_root_path_, + const String & object_storage_root_path_, const String & log_name, - MetadataStoragePtr && metadata_storage_, - ObjectStoragePtr && object_storage_, + MetadataStoragePtr metadata_storage_, + ObjectStoragePtr object_storage_, DiskType disk_type_, bool send_metadata_, - uint64_t thread_pool_size) - : IDisk(std::make_unique(log_name, thread_pool_size)) + uint64_t thread_pool_size_) + : IDisk(std::make_unique(log_name, thread_pool_size_)) , name(name_) - , remote_fs_root_path(remote_fs_root_path_) - , log (&Poco::Logger::get(log_name)) + , object_storage_root_path(object_storage_root_path_) + , log (&Poco::Logger::get("DiskObjectStorage(" + log_name + ")")) , disk_type(disk_type_) , metadata_storage(std::move(metadata_storage_)) , object_storage(std::move(object_storage_)) , send_metadata(send_metadata_) + , threadpool_size(thread_pool_size_) , metadata_helper(std::make_unique(this, ReadSettings{})) {} -PathsWithSize DiskObjectStorage::getObjectStoragePaths(const String & local_path) const +StoredObjects DiskObjectStorage::getStorageObjects(const String & local_path) const { - return metadata_storage->getObjectStoragePaths(local_path); + return metadata_storage->getStorageObjects(local_path); } -void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std::vector & paths_map) +void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std::vector & paths_map) { /// Protect against concurrent delition of files (for example because of a merge). if (metadata_storage->isFile(local_path)) { try { - paths_map.emplace_back(local_path, getObjectStoragePaths(local_path)); + paths_map.emplace_back(local_path, getStorageObjects(local_path)); } catch (const Exception & e) { @@ -247,25 +247,20 @@ std::unordered_map DiskObjectStorage::getSerializedMetadata(cons String DiskObjectStorage::getUniqueId(const String & path) const { - LOG_TRACE(log, "Remote path: {}, Path: {}", remote_fs_root_path, path); String id; - auto blobs_paths = metadata_storage->getObjectStoragePaths(path); + auto blobs_paths = metadata_storage->getStorageObjects(path); if (!blobs_paths.empty()) - id = blobs_paths[0].path; + id = blobs_paths[0].absolute_path; return id; } -bool DiskObjectStorage::checkObjectExists(const String & path) const -{ - if (!path.starts_with(remote_fs_root_path)) - return false; - - return object_storage->exists(path); -} - bool DiskObjectStorage::checkUniqueId(const String & id) const { - return checkObjectExists(id); + if (!id.starts_with(object_storage_root_path)) + return false; + + auto object = StoredObject::create(*object_storage, id, {}, true); + return object_storage->exists(object); } void DiskObjectStorage::createHardLink(const String & src_path, const String & dst_path, bool should_send_metadata) @@ -394,7 +389,8 @@ ReservationPtr DiskObjectStorage::reserve(UInt64 bytes) if (!tryReserve(bytes)) return {}; - return std::make_unique(std::static_pointer_cast(shared_from_this()), bytes); + return std::make_unique( + std::static_pointer_cast(shared_from_this()), bytes); } void DiskObjectStorage::removeSharedFileIfExists(const String & path, bool delete_metadata_only) @@ -404,7 +400,8 @@ void DiskObjectStorage::removeSharedFileIfExists(const String & path, bool delet transaction->commit(); } -void DiskObjectStorage::removeSharedRecursive(const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) +void DiskObjectStorage::removeSharedRecursive( + const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) { auto transaction = createObjectStorageTransaction(); transaction->removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only); @@ -437,6 +434,24 @@ std::optional DiskObjectStorage::tryReserve(UInt64 bytes) return {}; } +bool DiskObjectStorage::supportsCache() const +{ + return object_storage->supportsCache(); +} + +DiskObjectStoragePtr DiskObjectStorage::createDiskObjectStorage(const String & name_) +{ + return std::make_shared( + name_, + object_storage_root_path, + name, + metadata_storage, + object_storage, + disk_type, + send_metadata, + threadpool_size); +} + std::unique_ptr DiskObjectStorage::readFile( const String & path, const ReadSettings & settings, @@ -444,7 +459,7 @@ std::unique_ptr DiskObjectStorage::readFile( std::optional file_size) const { return object_storage->readObjects( - metadata_storage->getObjectStoragePaths(path), + metadata_storage->getStorageObjects(path), settings, read_hint, file_size); @@ -456,13 +471,16 @@ std::unique_ptr DiskObjectStorage::writeFile( WriteMode mode, const WriteSettings & settings) { + LOG_TEST(log, "Write file: {}", path); + auto transaction = createObjectStorageTransaction(); auto result = transaction->writeFile(path, buf_size, mode, settings); return result; } -void DiskObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context_, const String &, const DisksMap &) +void DiskObjectStorage::applyNewSettings( + const Poco::Util::AbstractConfiguration & config, ContextPtr context_, const String &, const DisksMap &) { const auto config_prefix = "storage_configuration.disks." + name; object_storage->applyNewSettings(config, config_prefix, context_); @@ -471,13 +489,15 @@ void DiskObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration exec->setMaxThreads(config.getInt(config_prefix + ".thread_pool_size", 16)); } -void DiskObjectStorage::restoreMetadataIfNeeded(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) +void DiskObjectStorage::restoreMetadataIfNeeded( + const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) { if (send_metadata) { metadata_helper->restore(config, config_prefix, context); - if (metadata_helper->readSchemaVersion(object_storage.get(), remote_fs_root_path) < DiskObjectStorageRemoteMetadataRestoreHelper::RESTORABLE_SCHEMA_VERSION) + auto current_schema_version = metadata_helper->readSchemaVersion(object_storage.get(), object_storage_root_path); + if (current_schema_version < DiskObjectStorageRemoteMetadataRestoreHelper::RESTORABLE_SCHEMA_VERSION) metadata_helper->migrateToRestorableSchema(); metadata_helper->findLastRevision(); diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index 56a1f7b7a3f..9494f421e6a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -29,13 +30,13 @@ friend class DiskObjectStorageRemoteMetadataRestoreHelper; public: DiskObjectStorage( const String & name_, - const String & remote_fs_root_path_, + const String & object_storage_root_path_, const String & log_name, - MetadataStoragePtr && metadata_storage_, - ObjectStoragePtr && object_storage_, + MetadataStoragePtr metadata_storage_, + ObjectStoragePtr object_storage_, DiskType disk_type_, bool send_metadata_, - uint64_t thread_pool_size); + uint64_t thread_pool_size_); /// Create fake transaction DiskTransactionPtr createTransaction() override; @@ -50,9 +51,9 @@ public: const String & getPath() const override { return metadata_storage->getPath(); } - PathsWithSize getObjectStoragePaths(const String & local_path) const override; + StoredObjects getStorageObjects(const String & local_path) const override; - void getRemotePathsRecursive(const String & local_path, std::vector & paths_map) override; + void getRemotePathsRecursive(const String & local_path, std::vector & paths_map) override; std::string getCacheBasePath() const override { @@ -104,7 +105,6 @@ public: String getUniqueId(const String & path) const override; - bool checkObjectExists(const String & path) const; bool checkUniqueId(const String & id) const override; void createHardLink(const String & src_path, const String & dst_path) override; @@ -163,6 +163,11 @@ public: void syncRevision(UInt64 revision) override; UInt64 getRevision() const override; + + DiskObjectStoragePtr createDiskObjectStorage(const String & name_) override; + + bool supportsCache() const override; + private: /// Create actual disk object storage transaction for operations @@ -170,7 +175,7 @@ private: DiskTransactionPtr createObjectStorageTransaction(); const String name; - const String remote_fs_root_path; + const String object_storage_root_path; Poco::Logger * log; const DiskType disk_type; @@ -184,10 +189,13 @@ private: std::optional tryReserve(UInt64 bytes); const bool send_metadata; + size_t threadpool_size; std::unique_ptr metadata_helper; }; +using DiskObjectStoragePtr = std::shared_ptr; + class DiskObjectStorageReservation final : public IReservation { public: @@ -210,7 +218,7 @@ public: ~DiskObjectStorageReservation() override; private: - std::shared_ptr disk; + DiskObjectStoragePtr disk; UInt64 size; UInt64 unreserved_space; CurrentMetrics::Increment metric_increment; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp index a2c0c8abd36..ae03915d944 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include namespace DB { @@ -36,23 +38,24 @@ void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf) for (size_t i = 0; i < storage_objects_count; ++i) { - String remote_fs_object_path; - size_t remote_fs_object_size; - readIntText(remote_fs_object_size, buf); + String object_relative_path; + size_t object_size; + readIntText(object_size, buf); assertChar('\t', buf); - readEscapedString(remote_fs_object_path, buf); + readEscapedString(object_relative_path, buf); if (version == VERSION_ABSOLUTE_PATHS) { - if (!remote_fs_object_path.starts_with(remote_fs_root_path)) + if (!object_relative_path.starts_with(object_storage_root_path)) throw Exception(ErrorCodes::UNKNOWN_FORMAT, "Path in metadata does not correspond to root path. Path: {}, root path: {}, disk path: {}", - remote_fs_object_path, remote_fs_root_path, common_metadata_path); + object_relative_path, object_storage_root_path, common_metadata_path); - remote_fs_object_path = remote_fs_object_path.substr(remote_fs_root_path.size()); + object_relative_path = object_relative_path.substr(object_storage_root_path.size()); } assertChar('\n', buf); - storage_objects[i].path = remote_fs_object_path; - storage_objects[i].bytes_size = remote_fs_object_size; + + storage_objects[i].relative_path = object_relative_path; + storage_objects[i].bytes_size = object_size; } readIntText(ref_count, buf); @@ -81,11 +84,11 @@ void DiskObjectStorageMetadata::serialize(WriteBuffer & buf, bool sync) const writeIntText(total_size, buf); writeChar('\n', buf); - for (const auto & [remote_fs_object_path, remote_fs_object_size] : storage_objects) + for (const auto & [object_relative_path, object_size] : storage_objects) { - writeIntText(remote_fs_object_size, buf); + writeIntText(object_size, buf); writeChar('\t', buf); - writeEscapedString(remote_fs_object_path, buf); + writeEscapedString(object_relative_path, buf); writeChar('\n', buf); } @@ -110,17 +113,17 @@ std::string DiskObjectStorageMetadata::serializeToString() const /// Load metadata by path or create empty if `create` flag is set. DiskObjectStorageMetadata::DiskObjectStorageMetadata( const std::string & common_metadata_path_, - const String & remote_fs_root_path_, + const String & object_storage_root_path_, const String & metadata_file_path_) : common_metadata_path(common_metadata_path_) - , remote_fs_root_path(remote_fs_root_path_) + , object_storage_root_path(object_storage_root_path_) , metadata_file_path(metadata_file_path_) { } void DiskObjectStorageMetadata::addObject(const String & path, size_t size) { - if (!remote_fs_root_path.empty() && path.starts_with(remote_fs_root_path)) + if (!object_storage_root_path.empty() && path.starts_with(object_storage_root_path)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected relative path"); total_size += size; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h index 0f2a2a5507d..d3ea5795dd3 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h @@ -22,8 +22,7 @@ private: /// Relative paths of blobs. RelativePathsWithSize storage_objects; - /// URI - const std::string & remote_fs_root_path; + const std::string object_storage_root_path; /// Relative path to metadata file on local FS. const std::string metadata_file_path; @@ -44,7 +43,7 @@ public: DiskObjectStorageMetadata( const std::string & common_metadata_path_, - const std::string & remote_fs_root_path_, + const std::string & object_storage_root_path_, const std::string & metadata_file_path_); void addObject(const std::string & path, size_t size); @@ -57,7 +56,7 @@ public: std::string getBlobsCommonPrefix() const { - return remote_fs_root_path; + return object_storage_root_path; } RelativePathsWithSize getBlobsRelativePaths() const diff --git a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp index c9dbb5de078..bce8d6cf7d0 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -23,10 +24,12 @@ static String revisionToString(UInt64 revision) return std::bitset<64>(revision).to_string(); } -void DiskObjectStorageRemoteMetadataRestoreHelper::createFileOperationObject(const String & operation_name, UInt64 revision, const ObjectAttributes & metadata) const +void DiskObjectStorageRemoteMetadataRestoreHelper::createFileOperationObject( + const String & operation_name, UInt64 revision, const ObjectAttributes & metadata) const { - const String path = disk->remote_fs_root_path + "operations/r" + revisionToString(revision) + operation_log_suffix + "-" + operation_name; - auto buf = disk->object_storage->writeObject(path, WriteMode::Rewrite, metadata); + const String relative_path = "operations/r" + revisionToString(revision) + operation_log_suffix + "-" + operation_name; + StoredObject object(fs::path(disk->object_storage_root_path) / relative_path); + auto buf = disk->object_storage->writeObject(object, WriteMode::Rewrite, metadata); buf->write('0'); buf->finalize(); } @@ -42,9 +45,12 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::findLastRevision() LOG_TRACE(disk->log, "Check object exists with revision prefix {}", revision_prefix); + const auto & object_storage = disk->object_storage; + StoredObject revision_object{disk->object_storage_root_path + "r" + revision_prefix}; + StoredObject revision_operation_object{disk->object_storage_root_path + "operations/r" + revision_prefix}; + /// Check file or operation with such revision prefix exists. - if (disk->object_storage->exists(disk->remote_fs_root_path + "r" + revision_prefix) - || disk->object_storage->exists(disk->remote_fs_root_path + "operations/r" + revision_prefix)) + if (object_storage->exists(revision_object) || object_storage->exists(revision_operation_object)) revision += "1"; else revision += "0"; @@ -55,12 +61,12 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::findLastRevision() int DiskObjectStorageRemoteMetadataRestoreHelper::readSchemaVersion(IObjectStorage * object_storage, const String & source_path) { - const std::string path = source_path + SCHEMA_VERSION_OBJECT; + StoredObject object(fs::path(source_path) / SCHEMA_VERSION_OBJECT); int version = 0; - if (!object_storage->exists(path)) + if (!object_storage->exists(object)) return version; - auto buf = object_storage->readObject(path); + auto buf = object_storage->readObject(object); readIntText(version, *buf); return version; @@ -68,9 +74,9 @@ int DiskObjectStorageRemoteMetadataRestoreHelper::readSchemaVersion(IObjectStora void DiskObjectStorageRemoteMetadataRestoreHelper::saveSchemaVersion(const int & version) const { - auto path = disk->remote_fs_root_path + SCHEMA_VERSION_OBJECT; + StoredObject object{fs::path(disk->object_storage_root_path) / SCHEMA_VERSION_OBJECT}; - auto buf = disk->object_storage->writeObject(path, WriteMode::Rewrite); + auto buf = disk->object_storage->writeObject(object, WriteMode::Rewrite); writeIntText(version, *buf); buf->finalize(); @@ -78,20 +84,21 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::saveSchemaVersion(const int & void DiskObjectStorageRemoteMetadataRestoreHelper::updateObjectMetadata(const String & key, const ObjectAttributes & metadata) const { - disk->object_storage->copyObject(key, key, metadata); + StoredObject object{key}; + disk->object_storage->copyObject(object, object, metadata); } void DiskObjectStorageRemoteMetadataRestoreHelper::migrateFileToRestorableSchema(const String & path) const { LOG_TRACE(disk->log, "Migrate file {} to restorable schema", disk->metadata_storage->getPath() + path); - auto objects = disk->metadata_storage->getObjectStoragePaths(path); - for (const auto & [object_path, _] : objects) + auto objects = disk->metadata_storage->getStorageObjects(path); + for (const auto & object : objects) { ObjectAttributes metadata { {"path", path} }; - updateObjectMetadata(object_path, metadata); + updateObjectMetadata(object.absolute_path, metadata); } } void DiskObjectStorageRemoteMetadataRestoreHelper::migrateToRestorableSchemaRecursive(const String & path, Futures & results) @@ -180,7 +187,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restore(const Poco::Util::Abs try { RestoreInformation information; - information.source_path = disk->remote_fs_root_path; + information.source_path = disk->object_storage_root_path; information.source_namespace = disk->object_storage->getObjectsNamespace(); readRestoreInformation(information); @@ -194,12 +201,14 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restore(const Poco::Util::Abs { /// In this case we need to additionally cleanup S3 from objects with later revision. /// Will be simply just restore to different path. - if (information.source_path == disk->remote_fs_root_path && information.revision != LATEST_REVISION) + if (information.source_path == disk->object_storage_root_path && information.revision != LATEST_REVISION) throw Exception("Restoring to the same bucket and path is allowed if revision is latest (0)", ErrorCodes::BAD_ARGUMENTS); /// This case complicates S3 cleanup in case of unsuccessful restore. - if (information.source_path != disk->remote_fs_root_path && disk->remote_fs_root_path.starts_with(information.source_path)) - throw Exception("Restoring to the same bucket is allowed only if source path is not a sub-path of configured path in S3 disk", ErrorCodes::BAD_ARGUMENTS); + if (information.source_path != disk->object_storage_root_path && disk->object_storage_root_path.starts_with(information.source_path)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Restoring to the same bucket is allowed only if source path is not a sub-path of configured path in S3 disk"); } else { @@ -215,7 +224,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restore(const Poco::Util::Abs LOG_INFO(disk->log, "Removing old metadata..."); - bool cleanup_s3 = information.source_path != disk->remote_fs_root_path; + bool cleanup_s3 = information.source_path != disk->object_storage_root_path; for (const auto & root : data_roots) if (disk->exists(root)) disk->removeSharedRecursive(root + '/', !cleanup_s3, {}); @@ -347,24 +356,24 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage * LOG_INFO(disk->log, "Starting restore files for disk {}", disk->name); std::vector> results; - auto restore_files = [this, &source_object_storage, &restore_information, &results](const PathsWithSize & keys) + auto restore_files = [this, &source_object_storage, &restore_information, &results](const RelativePathsWithSize & objects) { std::vector keys_names; - for (const auto & [key, size] : keys) + for (const auto & object : objects) { - LOG_INFO(disk->log, "Calling restore for key for disk {}", key); + LOG_INFO(disk->log, "Calling restore for key for disk {}", object.relative_path); /// Skip file operations objects. They will be processed separately. - if (key.find("/operations/") != String::npos) + if (object.relative_path.find("/operations/") != String::npos) continue; - const auto [revision, _] = extractRevisionAndOperationFromKey(key); + const auto [revision, _] = extractRevisionAndOperationFromKey(object.relative_path); /// Filter early if it's possible to get revision from key. if (revision > restore_information.revision) continue; - keys_names.push_back(key); + keys_names.push_back(object.relative_path); } if (!keys_names.empty()) @@ -394,7 +403,8 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage * } -void DiskObjectStorageRemoteMetadataRestoreHelper::processRestoreFiles(IObjectStorage * source_object_storage, const String & source_path, const std::vector & keys) const +void DiskObjectStorageRemoteMetadataRestoreHelper::processRestoreFiles( + IObjectStorage * source_object_storage, const String & source_path, const std::vector & keys) const { for (const auto & key : keys) { @@ -421,9 +431,12 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::processRestoreFiles(IObjectSt disk->createDirectories(directoryPath(path)); auto relative_key = shrinkKey(source_path, key); + StoredObject object_from{key}; + StoredObject object_to{fs::path(disk->object_storage_root_path) / relative_key}; + /// Copy object if we restore to different bucket / path. - if (source_object_storage->getObjectsNamespace() != disk->object_storage->getObjectsNamespace() || disk->remote_fs_root_path != source_path) - source_object_storage->copyObjectToAnotherObjectStorage(key, disk->remote_fs_root_path + relative_key, *disk->object_storage); + if (source_object_storage->getObjectsNamespace() != disk->object_storage->getObjectsNamespace() || disk->object_storage_root_path != source_path) + source_object_storage->copyObjectToAnotherObjectStorage(object_from, object_to, *disk->object_storage); auto tx = disk->metadata_storage->createTransaction(); tx->addBlobToMetadata(path, relative_key, meta.size_bytes); @@ -454,20 +467,21 @@ static String pathToDetached(const String & source_path) void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObjectStorage * source_object_storage, const RestoreInformation & restore_information) { /// Enable recording file operations if we restore to different bucket / path. - bool send_metadata = source_object_storage->getObjectsNamespace() != disk->object_storage->getObjectsNamespace() || disk->remote_fs_root_path != restore_information.source_path; + bool send_metadata = source_object_storage->getObjectsNamespace() != disk->object_storage->getObjectsNamespace() + || disk->object_storage_root_path != restore_information.source_path; std::set renames; - auto restore_file_operations = [this, &source_object_storage, &restore_information, &renames, &send_metadata](const PathsWithSize & keys) + auto restore_file_operations = [this, &source_object_storage, &restore_information, &renames, &send_metadata](const RelativePathsWithSize & objects) { const String rename = "rename"; const String hardlink = "hardlink"; - for (const auto & [key, _]: keys) + for (const auto & object : objects) { - const auto [revision, operation] = extractRevisionAndOperationFromKey(key); + const auto [revision, operation] = extractRevisionAndOperationFromKey(object.relative_path); if (revision == UNKNOWN_REVISION) { - LOG_WARNING(disk->log, "Skip key {} with unknown revision", key); + LOG_WARNING(disk->log, "Skip key {} with unknown revision", object.relative_path); continue; } @@ -480,7 +494,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject if (send_metadata) revision_counter = revision - 1; - auto object_attributes = *(source_object_storage->getObjectMetadata(key).attributes); + auto object_attributes = *(source_object_storage->getObjectMetadata(object.relative_path).attributes); if (operation == rename) { auto from_path = object_attributes["from_path"]; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 3ca85282e25..6e807747478 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -22,12 +22,10 @@ namespace ErrorCodes DiskObjectStorageTransaction::DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, - const std::string & remote_fs_root_path_, DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_) : object_storage(object_storage_) , metadata_storage(metadata_storage_) , metadata_transaction(metadata_storage.createTransaction()) - , remote_fs_root_path(remote_fs_root_path_) , metadata_helper(metadata_helper_) {} @@ -59,15 +57,17 @@ struct PureMetadataObjectStorageOperation final : public IDiskObjectStorageOpera void finalize() override { } + + std::string getInfoForLog() const override { return fmt::format("PureMetadataObjectStorageOperation"); } }; struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation { std::string path; bool delete_metadata_only; - bool remove_from_cache{false}; - PathsWithSize paths_to_remove; + StoredObjects objects_to_remove; bool if_exists; + bool remove_from_cache = false; RemoveObjectStorageOperation( IObjectStorage & object_storage_, @@ -81,6 +81,11 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation , if_exists(if_exists_) {} + std::string getInfoForLog() const override + { + return fmt::format("RemoveObjectStorageOperation (path: {}, if exists: {})", path, if_exists); + } + void execute(MetadataTransactionPtr tx) override { if (!metadata_storage.exists(path)) @@ -97,13 +102,13 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation try { uint32_t hardlink_count = metadata_storage.getHardlinkCount(path); - auto objects = metadata_storage.getObjectStoragePaths(path); + auto objects = metadata_storage.getStorageObjects(path); tx->unlinkMetadata(path); if (hardlink_count == 0) { - paths_to_remove = objects; + objects_to_remove = objects; remove_from_cache = true; } } @@ -129,25 +134,24 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation void finalize() override { - if (!delete_metadata_only && !paths_to_remove.empty()) - object_storage.removeObjects(paths_to_remove); + if (!delete_metadata_only && !objects_to_remove.empty()) + object_storage.removeObjects(objects_to_remove); if (remove_from_cache) { - for (const auto & path_to_remove : paths_to_remove) - object_storage.removeFromCache(path_to_remove.path); + for (const auto & object : objects_to_remove) + object_storage.removeCacheIfExists(object.getPathKeyForCache()); } - } }; struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOperation { std::string path; - std::unordered_map paths_to_remove; + std::unordered_map objects_to_remove; bool keep_all_batch_data; NameSet file_names_remove_metadata_only; - PathsWithSize path_to_remove_from_cache; + StoredObjects objects_to_remove_from_cache; RemoveRecursiveObjectStorageOperation( IObjectStorage & object_storage_, @@ -161,6 +165,11 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp , file_names_remove_metadata_only(file_names_remove_metadata_only_) {} + std::string getInfoForLog() const override + { + return fmt::format("RemoveRecursiveObjectStorageOperation (path: {})", path); + } + void removeMetadataRecursive(MetadataTransactionPtr tx, const std::string & path_to_remove) { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. @@ -170,14 +179,14 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp try { uint32_t hardlink_count = metadata_storage.getHardlinkCount(path_to_remove); - auto objects_paths = metadata_storage.getObjectStoragePaths(path_to_remove); + auto objects_paths = metadata_storage.getStorageObjects(path_to_remove); tx->unlinkMetadata(path_to_remove); if (hardlink_count == 0) { - paths_to_remove[path_to_remove] = objects_paths; - path_to_remove_from_cache.insert(path_to_remove_from_cache.end(), objects_paths.begin(), objects_paths.end()); + objects_to_remove[path_to_remove] = objects_paths; + objects_to_remove_from_cache.insert(objects_to_remove_from_cache.end(), objects_paths.begin(), objects_paths.end()); } } @@ -218,8 +227,8 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp { if (!keep_all_batch_data) { - PathsWithSize remove_from_remote; - for (auto && [local_path, remote_paths] : paths_to_remove) + StoredObjects remove_from_remote; + for (auto && [local_path, remote_paths] : objects_to_remove) { if (!file_names_remove_metadata_only.contains(fs::path(local_path).filename())) { @@ -229,8 +238,8 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp object_storage.removeObjects(remove_from_remote); } - for (const auto & [path_to_remove, _] : path_to_remove_from_cache) - object_storage.removeFromCache(path_to_remove); + for (const auto & object : objects_to_remove_from_cache) + object_storage.removeCacheIfExists(object.getPathKeyForCache()); } }; @@ -239,7 +248,7 @@ struct ReplaceFileObjectStorageOperation final : public IDiskObjectStorageOperat { std::string path_from; std::string path_to; - PathsWithSize blobs_to_remove; + StoredObjects objects_to_remove; ReplaceFileObjectStorageOperation( IObjectStorage & object_storage_, @@ -251,11 +260,16 @@ struct ReplaceFileObjectStorageOperation final : public IDiskObjectStorageOperat , path_to(path_to_) {} + std::string getInfoForLog() const override + { + return fmt::format("ReplaceFileObjectStorageOperation (path_from: {}, path_to: {})", path_from, path_to); + } + void execute(MetadataTransactionPtr tx) override { if (metadata_storage.exists(path_to)) { - blobs_to_remove = metadata_storage.getObjectStoragePaths(path_to); + objects_to_remove = metadata_storage.getStorageObjects(path_to); tx->replaceFile(path_from, path_to); } else @@ -269,28 +283,29 @@ struct ReplaceFileObjectStorageOperation final : public IDiskObjectStorageOperat void finalize() override { - if (!blobs_to_remove.empty()) - object_storage.removeObjects(blobs_to_remove); + if (!objects_to_remove.empty()) + object_storage.removeObjects(objects_to_remove); } }; struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperation { - std::string path; - std::string blob_path; - size_t size; + StoredObject object; std::function on_execute; WriteFileObjectStorageOperation( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, - const std::string & path_, - const std::string & blob_path_) + const StoredObject & object_) : IDiskObjectStorageOperation(object_storage_, metadata_storage_) - , path(path_) - , blob_path(blob_path_) + , object(object_) {} + std::string getInfoForLog() const override + { + return fmt::format("WriteFileObjectStorageOperation"); + } + void setOnExecute(std::function && on_execute_) { on_execute = on_execute_; @@ -304,8 +319,8 @@ struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperatio void undo() override { - if (object_storage.exists(blob_path)) - object_storage.removeObject(blob_path); + if (object_storage.exists(object)) + object_storage.removeObject(object); } void finalize() override @@ -316,47 +331,50 @@ struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperatio struct CopyFileObjectStorageOperation final : public IDiskObjectStorageOperation { + /// Local paths std::string from_path; std::string to_path; - std::string remote_fs_root_path; - std::vector created_blobs; + StoredObjects created_objects; CopyFileObjectStorageOperation( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, const std::string & from_path_, - const std::string & to_path_, - const std::string & remote_fs_root_path_) + const std::string & to_path_) : IDiskObjectStorageOperation(object_storage_, metadata_storage_) , from_path(from_path_) , to_path(to_path_) - , remote_fs_root_path(remote_fs_root_path_) {} + std::string getInfoForLog() const override + { + return fmt::format("CopyFileObjectStorageOperation (path_from: {}, path_to: {})", from_path, to_path); + } + void execute(MetadataTransactionPtr tx) override { tx->createEmptyMetadataFile(to_path); - auto source_blobs = metadata_storage.getObjectStoragePaths(from_path); /// Full paths + auto source_blobs = metadata_storage.getStorageObjects(from_path); /// Full paths - for (const auto & [blob_from, size] : source_blobs) + for (const auto & object_from : source_blobs) { - auto blob_name = getRandomASCIIString(); + std::string blob_name = object_storage.generateBlobNameForPath(to_path); + auto object_to = StoredObject::create( + object_storage, fs::path(metadata_storage.getObjectStorageRootPath()) / blob_name); - auto blob_to = fs::path(remote_fs_root_path) / blob_name; + object_storage.copyObject(object_from, object_to); - object_storage.copyObject(blob_from, blob_to); + tx->addBlobToMetadata(to_path, blob_name, object_from.bytes_size); - tx->addBlobToMetadata(to_path, blob_name, size); - - created_blobs.push_back(blob_to); + created_objects.push_back(object_to); } } void undo() override { - for (const auto & blob_path : created_blobs) - object_storage.removeObject(blob_path); + for (const auto & object : created_objects) + object_storage.removeObject(object); } void finalize() override @@ -380,7 +398,7 @@ void DiskObjectStorageTransaction::createDirectories(const std::string & path) operations_to_execute.emplace_back( std::make_unique(object_storage, metadata_storage, [path](MetadataTransactionPtr tx) { - tx->createDicrectoryRecursive(path); + tx->createDirectoryRecursive(path); })); } @@ -411,7 +429,8 @@ void DiskObjectStorageTransaction::moveFile(const String & from_path, const Stri void DiskObjectStorageTransaction::replaceFile(const std::string & from_path, const std::string & to_path) { - operations_to_execute.emplace_back(std::make_unique(object_storage, metadata_storage, from_path, to_path)); + auto operation = std::make_unique(object_storage, metadata_storage, from_path, to_path); + operations_to_execute.emplace_back(std::move(operation)); } void DiskObjectStorageTransaction::clearDirectory(const std::string & path) @@ -430,17 +449,22 @@ void DiskObjectStorageTransaction::removeFile(const std::string & path) void DiskObjectStorageTransaction::removeSharedFile(const std::string & path, bool keep_shared_data) { - operations_to_execute.emplace_back(std::make_unique(object_storage, metadata_storage, path, keep_shared_data, false)); + auto operation = std::make_unique(object_storage, metadata_storage, path, keep_shared_data, false); + operations_to_execute.emplace_back(std::move(operation)); } -void DiskObjectStorageTransaction::removeSharedRecursive(const std::string & path, bool keep_all_shared_data, const NameSet & file_names_remove_metadata_only) +void DiskObjectStorageTransaction::removeSharedRecursive( + const std::string & path, bool keep_all_shared_data, const NameSet & file_names_remove_metadata_only) { - operations_to_execute.emplace_back(std::make_unique(object_storage, metadata_storage, path, keep_all_shared_data, file_names_remove_metadata_only)); + auto operation = std::make_unique( + object_storage, metadata_storage, path, keep_all_shared_data, file_names_remove_metadata_only); + operations_to_execute.emplace_back(std::move(operation)); } void DiskObjectStorageTransaction::removeSharedFileIfExists(const std::string & path, bool keep_shared_data) { - operations_to_execute.emplace_back(std::make_unique(object_storage, metadata_storage, path, keep_shared_data, true)); + auto operation = std::make_unique(object_storage, metadata_storage, path, keep_shared_data, true); + operations_to_execute.emplace_back(std::move(operation)); } void DiskObjectStorageTransaction::removeDirectory(const std::string & path) @@ -464,7 +488,8 @@ void DiskObjectStorageTransaction::removeFileIfExists(const std::string & path) } -void DiskObjectStorageTransaction::removeSharedFiles(const RemoveBatchRequest & files, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) +void DiskObjectStorageTransaction::removeSharedFiles( + const RemoveBatchRequest & files, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) { for (const auto & file : files) { @@ -493,9 +518,10 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile const WriteSettings & settings, bool autocommit) { - auto blob_name = getRandomASCIIString(); - + String blob_name; std::optional object_attributes; + + blob_name = object_storage.generateBlobNameForPath(path); if (metadata_helper) { auto revision = metadata_helper->revision_counter + 1; @@ -506,9 +532,8 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; } - auto blob_path = fs::path(remote_fs_root_path) / blob_name; - - auto write_operation = std::make_unique(object_storage, metadata_storage, path, blob_path); + auto object = StoredObject::create(object_storage, fs::path(metadata_storage.getObjectStorageRootPath()) / blob_name); + auto write_operation = std::make_unique(object_storage, metadata_storage, object); std::function create_metadata_callback; if (autocommit) @@ -552,9 +577,12 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile /// We always use mode Rewrite because we simulate append using metadata and different files return object_storage.writeObject( - blob_path, WriteMode::Rewrite, object_attributes, + object, + WriteMode::Rewrite, + object_attributes, std::move(create_metadata_callback), - buf_size, settings); + buf_size, + settings); } @@ -596,7 +624,8 @@ void DiskObjectStorageTransaction::createFile(const std::string & path) void DiskObjectStorageTransaction::copyFile(const std::string & from_file_path, const std::string & to_file_path) { - operations_to_execute.emplace_back(std::make_unique(object_storage, metadata_storage, from_file_path, to_file_path, remote_fs_root_path)); + operations_to_execute.emplace_back( + std::make_unique(object_storage, metadata_storage, from_file_path, to_file_path)); } void DiskObjectStorageTransaction::commit() @@ -609,7 +638,7 @@ void DiskObjectStorageTransaction::commit() } catch (Exception & ex) { - ex.addMessage(fmt::format("While executing operation #{}", i)); + ex.addMessage(fmt::format("While executing operation #{} ({})", i, operations_to_execute[i]->getInfoForLog())); for (int64_t j = i; j >= 0; --j) { diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 362b5404707..ceed79a23b7 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -30,12 +30,15 @@ public: /// like removal of blobs. Such implementation can lead to garbage. virtual void finalize() = 0; virtual ~IDiskObjectStorageOperation() = default; + + virtual std::string getInfoForLog() const = 0; }; using DiskObjectStorageOperation = std::unique_ptr; using DiskObjectStorageOperations = std::vector; + /// Disk object storage transaction, actually implement some part of disk object storage /// logic. Works on top of non atomic operations with blobs and possibly atomic implementation /// of metadata storage. @@ -52,17 +55,18 @@ struct DiskObjectStorageTransaction final : public IDiskTransaction, std::enable private: IObjectStorage & object_storage; IMetadataStorage & metadata_storage; + MetadataTransactionPtr metadata_transaction; + /// TODO we can get rid of this params - const std::string & remote_fs_root_path; DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper; DiskObjectStorageOperations operations_to_execute; + public: DiskObjectStorageTransaction( IObjectStorage & object_storage_, IMetadataStorage & metadata_storage_, - const std::string & remote_fs_root_path_, DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper_); void commit() override; diff --git a/src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.cpp b/src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.cpp new file mode 100644 index 00000000000..a6b2f6b5f02 --- /dev/null +++ b/src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.cpp @@ -0,0 +1,280 @@ +#include "FakeMetadataStorageFromDisk.h" +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; + extern const int FS_METADATA_ERROR; +} + +FakeMetadataStorageFromDisk::FakeMetadataStorageFromDisk( + DiskPtr disk_, + ObjectStoragePtr object_storage_, + const std::string & object_storage_root_path_) + : disk(disk_) + , object_storage(object_storage_) + , object_storage_root_path(object_storage_root_path_) +{ +} + +MetadataTransactionPtr FakeMetadataStorageFromDisk::createTransaction() const +{ + return std::make_shared(*this, disk); +} + +const std::string & FakeMetadataStorageFromDisk::getPath() const +{ + return disk->getPath(); +} + +bool FakeMetadataStorageFromDisk::exists(const std::string & path) const +{ + return disk->exists(path); +} + +bool FakeMetadataStorageFromDisk::isFile(const std::string & path) const +{ + return disk->isFile(path); +} + +bool FakeMetadataStorageFromDisk::isDirectory(const std::string & path) const +{ + return disk->isDirectory(path); +} + +Poco::Timestamp FakeMetadataStorageFromDisk::getLastModified(const std::string & path) const +{ + return disk->getLastModified(path); +} + +time_t FakeMetadataStorageFromDisk::getLastChanged(const std::string & path) const +{ + return disk->getLastChanged(path); +} + +uint64_t FakeMetadataStorageFromDisk::getFileSize(const String & path) const +{ + return disk->getFileSize(path); +} + +std::vector FakeMetadataStorageFromDisk::listDirectory(const std::string & path) const +{ + std::vector result; + auto it = disk->iterateDirectory(path); + while (it->isValid()) + { + result.push_back(it->path()); + it->next(); + } + return result; +} + +DirectoryIteratorPtr FakeMetadataStorageFromDisk::iterateDirectory(const std::string & path) const +{ + return disk->iterateDirectory(path); +} + +std::string FakeMetadataStorageFromDisk::readFileToString(const std::string &) const +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "readFileToString is not implemented for FakeMetadataStorageFromDisk"); +} + +std::unordered_map FakeMetadataStorageFromDisk::getSerializedMetadata(const std::vector &) const +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getSerializedMetadata is not implemented for FakeMetadataStorageFromDisk"); +} + +StoredObjects FakeMetadataStorageFromDisk::getStorageObjects(const std::string & path) const +{ + std::string blob_name = object_storage->generateBlobNameForPath(path); + + std::string object_path = fs::path(object_storage_root_path) / blob_name; + size_t object_size = getFileSize(object_path); + + auto object = StoredObject::create(*object_storage, object_path, object_size); + return {std::move(object)}; +} + +uint32_t FakeMetadataStorageFromDisk::getHardlinkCount(const std::string & path) const +{ + size_t ref_count = disk->getRefCount(path); + assert(ref_count > 0); + return ref_count - 1; +} + +const IMetadataStorage & FakeMetadataStorageFromDiskTransaction::getStorageForNonTransactionalReads() const +{ + return metadata_storage; +} + +void FakeMetadataStorageFromDiskTransaction::addOperation(MetadataOperationPtr && operation) +{ + if (state != MetadataFromDiskTransactionState::PREPARING) + throw Exception( + ErrorCodes::FS_METADATA_ERROR, + "Cannot add operations to transaction in {} state, it should be in {} state", + toString(state), toString(MetadataFromDiskTransactionState::PREPARING)); + + operations.emplace_back(std::move(operation)); +} + +void FakeMetadataStorageFromDiskTransaction::commit() +{ + if (state != MetadataFromDiskTransactionState::PREPARING) + throw Exception( + ErrorCodes::FS_METADATA_ERROR, + "Cannot commit transaction in {} state, it should be in {} state", + toString(state), toString(MetadataFromDiskTransactionState::PREPARING)); + + { + std::unique_lock lock(metadata_storage.metadata_mutex); + for (size_t i = 0; i < operations.size(); ++i) + { + try + { + operations[i]->execute(); + } + catch (Exception & ex) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + ex.addMessage(fmt::format("While committing metadata operation #{}", i)); + state = MetadataFromDiskTransactionState::FAILED; + rollback(i); + throw; + } + } + } + + /// Do it in "best effort" mode + for (size_t i = 0; i < operations.size(); ++i) + { + try + { + operations[i]->finalize(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__, fmt::format("Failed to finalize operation #{}", i)); + } + } + + state = MetadataFromDiskTransactionState::COMMITTED; +} + +void FakeMetadataStorageFromDiskTransaction::rollback(size_t until_pos) +{ + /// Otherwise everything is alright + if (state == MetadataFromDiskTransactionState::FAILED) + { + for (int64_t i = until_pos; i >= 0; --i) + { + try + { + operations[i]->undo(); + } + catch (Exception & ex) + { + state = MetadataFromDiskTransactionState::PARTIALLY_ROLLED_BACK; + ex.addMessage(fmt::format("While rolling back operation #{}", i)); + throw; + } + } + } + else + { + /// Nothing to do, transaction committed or not even started to commit + } +} + +void FakeMetadataStorageFromDiskTransaction::writeStringToFile(const std::string & path, const std::string & data) +{ + auto wb = disk->writeFile(path); + wb->write(data.data(), data.size()); + wb->finalize(); +} + +void FakeMetadataStorageFromDiskTransaction::setLastModified(const std::string & path, const Poco::Timestamp & timestamp) +{ + disk->setLastModified(path, timestamp); +} + +void FakeMetadataStorageFromDiskTransaction::unlinkFile(const std::string & path) +{ + disk->removeFile(path); +} + +void FakeMetadataStorageFromDiskTransaction::removeRecursive(const std::string & path) +{ + disk->removeRecursive(path); +} + +void FakeMetadataStorageFromDiskTransaction::createDirectory(const std::string & path) +{ + disk->createDirectory(path); +} + +void FakeMetadataStorageFromDiskTransaction::createDirectoryRecursive(const std::string & path) +{ + disk->createDirectories(path); +} + +void FakeMetadataStorageFromDiskTransaction::removeDirectory(const std::string & path) +{ + disk->removeDirectory(path); +} + +void FakeMetadataStorageFromDiskTransaction::moveFile(const std::string & path_from, const std::string & path_to) +{ + disk->moveFile(path_from, path_to); +} + +void FakeMetadataStorageFromDiskTransaction::moveDirectory(const std::string & path_from, const std::string & path_to) +{ + disk->moveDirectory(path_from, path_to); +} + +void FakeMetadataStorageFromDiskTransaction::replaceFile(const std::string & path_from, const std::string & path_to) +{ + disk->replaceFile(path_from, path_to); +} + +void FakeMetadataStorageFromDiskTransaction::setReadOnly(const std::string & path) +{ + disk->setReadOnly(path); +} + +void FakeMetadataStorageFromDiskTransaction::createHardLink(const std::string & path_from, const std::string & path_to) +{ + disk->createHardLink(path_from, path_to); +} + +void FakeMetadataStorageFromDiskTransaction::createEmptyMetadataFile(const std::string & /* path */) +{ + /// Noop. +} + +void FakeMetadataStorageFromDiskTransaction::createMetadataFile( + const std::string & /* path */, const std::string & /* blob_name */, uint64_t /* size_in_bytes */) +{ + /// Noop. +} + +void FakeMetadataStorageFromDiskTransaction::addBlobToMetadata( + const std::string & /* path */, const std::string & /* blob_name */, uint64_t /* size_in_bytes */) +{ + /// Noop, local metadata files is only one file, it is the metadata file itself. +} + +void FakeMetadataStorageFromDiskTransaction::unlinkMetadata(const std::string & path) +{ + disk->removeFile(path); +} + +} diff --git a/src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.h b/src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.h new file mode 100644 index 00000000000..3fc223ea75b --- /dev/null +++ b/src/Disks/ObjectStorages/FakeMetadataStorageFromDisk.h @@ -0,0 +1,120 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class FakeMetadataStorageFromDisk final : public IMetadataStorage +{ +private: + friend class FakeMetadataStorageFromDiskTransaction; + + mutable std::shared_mutex metadata_mutex; + + DiskPtr disk; + ObjectStoragePtr object_storage; + std::string object_storage_root_path; + +public: + FakeMetadataStorageFromDisk( + DiskPtr disk_, + ObjectStoragePtr object_storage_, + const std::string & object_storage_root_path_); + + MetadataTransactionPtr createTransaction() const override; + + const std::string & getPath() const override; + + bool exists(const std::string & path) const override; + + bool isFile(const std::string & path) const override; + + bool isDirectory(const std::string & path) const override; + + uint64_t getFileSize(const String & path) const override; + + Poco::Timestamp getLastModified(const std::string & path) const override; + + time_t getLastChanged(const std::string & path) const override; + + std::vector listDirectory(const std::string & path) const override; + + DirectoryIteratorPtr iterateDirectory(const std::string & path) const override; + + std::string readFileToString(const std::string & path) const override; + + std::unordered_map getSerializedMetadata(const std::vector & file_paths) const override; + + uint32_t getHardlinkCount(const std::string & path) const override; + + DiskPtr getDisk() const { return disk; } + + StoredObjects getStorageObjects(const std::string & path) const override; + + std::string getObjectStorageRootPath() const override { return object_storage_root_path; } +}; + +class FakeMetadataStorageFromDiskTransaction final : public IMetadataTransaction +{ +private: + DiskPtr disk; + const FakeMetadataStorageFromDisk & metadata_storage; + + std::vector operations; + MetadataFromDiskTransactionState state{MetadataFromDiskTransactionState::PREPARING}; + + void addOperation(MetadataOperationPtr && operation); + + void rollback(size_t until_pos); + +public: + FakeMetadataStorageFromDiskTransaction( + const FakeMetadataStorageFromDisk & metadata_storage_, DiskPtr disk_) + : disk(disk_) + , metadata_storage(metadata_storage_) + {} + + ~FakeMetadataStorageFromDiskTransaction() override = default; + + const IMetadataStorage & getStorageForNonTransactionalReads() const final; + + void commit() final; + + void writeStringToFile(const std::string & path, const std::string & data) override; + + void createEmptyMetadataFile(const std::string & path) override; + + void createMetadataFile(const std::string & path, const std::string & blob_name, uint64_t size_in_bytes) override; + + void addBlobToMetadata(const std::string & path, const std::string & blob_name, uint64_t size_in_bytes) override; + + void setLastModified(const std::string & path, const Poco::Timestamp & timestamp) override; + + void setReadOnly(const std::string & path) override; + + void unlinkFile(const std::string & path) override; + + void createDirectory(const std::string & path) override; + + void createDirectoryRecursive(const std::string & path) override; + + void removeDirectory(const std::string & path) override; + + void removeRecursive(const std::string & path) override; + + void createHardLink(const std::string & path_from, const std::string & path_to) override; + + void moveFile(const std::string & path_from, const std::string & path_to) override; + + void moveDirectory(const std::string & path_from, const std::string & path_to) override; + + void replaceFile(const std::string & path_from, const std::string & path_to) override; + + void unlinkMetadata(const std::string & path) override; +}; + +} diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp index bedd1a83df1..4ffbf5b2ceb 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #if USE_HDFS @@ -32,40 +33,46 @@ void HDFSObjectStorage::startup() { } -bool HDFSObjectStorage::exists(const std::string & hdfs_uri) const +std::string HDFSObjectStorage::generateBlobNameForPath(const std::string & /* path */) { - const size_t begin_of_path = hdfs_uri.find('/', hdfs_uri.find("//") + 2); - const String remote_fs_object_path = hdfs_uri.substr(begin_of_path); + return getRandomASCIIString(); +} + +bool HDFSObjectStorage::exists(const StoredObject & object) const +{ + const auto & path = object.absolute_path; + const size_t begin_of_path = path.find('/', path.find("//") + 2); + const String remote_fs_object_path = path.substr(begin_of_path); return (0 == hdfsExists(hdfs_fs.get(), remote_fs_object_path.c_str())); } -std::unique_ptr HDFSObjectStorage::readObject( /// NOLINT - const std::string & path, +std::unique_ptr HDFSObjectStorage::readObject( /// NOLINT + const StoredObject & object, const ReadSettings & read_settings, std::optional, std::optional) const { - return std::make_unique(path, path, config, read_settings.remote_fs_buffer_size); + return std::make_unique(object.absolute_path, object.absolute_path, config, patchSettings(read_settings)); } std::unique_ptr HDFSObjectStorage::readObjects( /// NOLINT - const PathsWithSize & paths_to_read, + const StoredObjects & objects, const ReadSettings & read_settings, std::optional, std::optional) const { - auto hdfs_impl = std::make_unique(config, paths_to_read, read_settings); + auto hdfs_impl = std::make_unique(config, objects, patchSettings(read_settings)); auto buf = std::make_unique(std::move(hdfs_impl)); return std::make_unique(std::move(buf), settings->min_bytes_for_seek); } std::unique_ptr HDFSObjectStorage::writeObject( /// NOLINT - const std::string & path, + const StoredObject & object, WriteMode mode, std::optional attributes, FinalizeCallback && finalize_callback, size_t buf_size, - const WriteSettings &) + const WriteSettings & write_settings) { if (attributes.has_value()) throw Exception( @@ -74,10 +81,10 @@ std::unique_ptr HDFSObjectStorage::writeObject( /// NOL /// Single O_WRONLY in libhdfs adds O_TRUNC auto hdfs_buffer = std::make_unique( - path, config, settings->replication, buf_size, + object.absolute_path, config, settings->replication, patchSettings(write_settings), buf_size, mode == WriteMode::Rewrite ? O_WRONLY : O_WRONLY | O_APPEND); - return std::make_unique(std::move(hdfs_buffer), std::move(finalize_callback), path); + return std::make_unique(std::move(hdfs_buffer), std::move(finalize_callback), object.absolute_path); } @@ -94,8 +101,9 @@ void HDFSObjectStorage::listPrefix(const std::string & path, RelativePathsWithSi } /// Remove file. Throws exception if file doesn't exists or it's a directory. -void HDFSObjectStorage::removeObject(const std::string & path) +void HDFSObjectStorage::removeObject(const StoredObject & object) { + const auto & path = object.absolute_path; const size_t begin_of_path = path.find('/', path.find("//") + 2); /// Add path from root to file name @@ -105,22 +113,22 @@ void HDFSObjectStorage::removeObject(const std::string & path) } -void HDFSObjectStorage::removeObjects(const PathsWithSize & paths) +void HDFSObjectStorage::removeObjects(const StoredObjects & objects) { - for (const auto & [path, _] : paths) - removeObject(path); + for (const auto & object : objects) + removeObject(object); } -void HDFSObjectStorage::removeObjectIfExists(const std::string & path) +void HDFSObjectStorage::removeObjectIfExists(const StoredObject & object) { - if (exists(path)) - removeObject(path); + if (exists(object)) + removeObject(object); } -void HDFSObjectStorage::removeObjectsIfExist(const PathsWithSize & paths) +void HDFSObjectStorage::removeObjectsIfExist(const StoredObjects & objects) { - for (const auto & [path, _] : paths) - removeObjectIfExists(path); + for (const auto & object : objects) + removeObjectIfExists(object); } ObjectMetadata HDFSObjectStorage::getObjectMetadata(const std::string &) const @@ -131,8 +139,8 @@ ObjectMetadata HDFSObjectStorage::getObjectMetadata(const std::string &) const } void HDFSObjectStorage::copyObject( /// NOLINT - const std::string & object_from, - const std::string & object_to, + const StoredObject & object_from, + const StoredObject & object_to, std::optional object_to_attributes) { if (object_to_attributes.has_value()) @@ -147,8 +155,9 @@ void HDFSObjectStorage::copyObject( /// NOLINT } -void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr) +void HDFSObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr context) { + applyRemoteThrottlingSettings(context); } std::unique_ptr HDFSObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration &, const std::string &, ContextPtr) diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h index 69878568548..4687d63c128 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h @@ -41,34 +41,34 @@ public: using SettingsPtr = std::unique_ptr; HDFSObjectStorage( - FileCachePtr && cache_, const String & hdfs_root_path_, SettingsPtr settings_, const Poco::Util::AbstractConfiguration & config_) - : IObjectStorage(std::move(cache_)) - , config(config_) + : config(config_) , hdfs_builder(createHDFSBuilder(hdfs_root_path_, config)) , hdfs_fs(createHDFSFS(hdfs_builder.get())) , settings(std::move(settings_)) {} - bool exists(const std::string & hdfs_uri) const override; + std::string getName() const override { return "HDFSObjectStorage"; } - std::unique_ptr readObject( /// NOLINT - const std::string & path, + bool exists(const StoredObject & object) const override; + + std::unique_ptr readObject( /// NOLINT + const StoredObject & object, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT - const PathsWithSize & paths_to_read, + const StoredObjects & objects, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; /// Open the file for write and return WriteBufferFromFileBase object. std::unique_ptr writeObject( /// NOLINT - const std::string & path, + const StoredObject & object, WriteMode mode, std::optional attributes = {}, FinalizeCallback && finalize_callback = {}, @@ -78,19 +78,19 @@ public: void listPrefix(const std::string & path, RelativePathsWithSize & children) const override; /// Remove file. Throws exception if file doesn't exists or it's a directory. - void removeObject(const std::string & path) override; + void removeObject(const StoredObject & object) override; - void removeObjects(const PathsWithSize & paths) override; + void removeObjects(const StoredObjects & objects) override; - void removeObjectIfExists(const std::string & path) override; + void removeObjectIfExists(const StoredObject & object) override; - void removeObjectsIfExist(const PathsWithSize & paths) override; + void removeObjectsIfExist(const StoredObjects & objects) override; ObjectMetadata getObjectMetadata(const std::string & path) const override; void copyObject( /// NOLINT - const std::string & object_from, - const std::string & object_to, + const StoredObject & object_from, + const StoredObject & object_to, std::optional object_to_attributes = {}) override; void shutdown() override; @@ -110,6 +110,10 @@ public: const std::string & config_prefix, ContextPtr context) override; + std::string generateBlobNameForPath(const std::string & path) override; + + bool isRemote() const override { return true; } + private: const Poco::Util::AbstractConfiguration & config; diff --git a/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp b/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp index 21641aca392..e0cea249d00 100644 --- a/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp +++ b/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp @@ -36,7 +36,7 @@ void registerDiskHDFS(DiskFactory & factory) /// FIXME Cache currently unsupported :( - ObjectStoragePtr hdfs_storage = std::make_unique(nullptr, uri, std::move(settings), config); + ObjectStoragePtr hdfs_storage = std::make_unique(uri, std::move(settings), config); auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context_); diff --git a/src/Disks/ObjectStorages/IMetadataStorage.h b/src/Disks/ObjectStorages/IMetadataStorage.h index 49f4d4bcc51..a941f1ca514 100644 --- a/src/Disks/ObjectStorages/IMetadataStorage.h +++ b/src/Disks/ObjectStorages/IMetadataStorage.h @@ -23,7 +23,7 @@ class IMetadataStorage; /// interface. This transaction is more like "batch operation" than real "transaction". /// /// But for better usability we can get MetadataStorage interface and use some read methods. -struct IMetadataTransaction : private boost::noncopyable +class IMetadataTransaction : private boost::noncopyable { public: virtual void commit() = 0; @@ -43,7 +43,7 @@ public: virtual void createDirectory(const std::string & path) = 0; - virtual void createDicrectoryRecursive(const std::string & path) = 0; + virtual void createDirectoryRecursive(const std::string & path) = 0; virtual void removeDirectory(const std::string & path) = 0; @@ -85,12 +85,16 @@ using MetadataTransactionPtr = std::shared_ptr; /// small amounts of data (strings). class IMetadataStorage : private boost::noncopyable { +friend class MetadataStorageFromDiskTransaction; + public: virtual MetadataTransactionPtr createTransaction() const = 0; - /// General purpose functions (similar to Disk) + /// Get metadata root path. virtual const std::string & getPath() const = 0; + /// ==== General purpose methods. Define properties of object storage file based on metadata files ==== + virtual bool exists(const std::string & path) const = 0; virtual bool isFile(const std::string & path) const = 0; @@ -119,9 +123,11 @@ public: /// Read multiple metadata files into strings and return mapping from file_path -> metadata virtual std::unordered_map getSerializedMetadata(const std::vector & file_paths) const = 0; - /// Return [(object_storage_path, size_in_bytes), ...] for metadata path - /// object_storage_path is a full path to the blob. - virtual PathsWithSize getObjectStoragePaths(const std::string & path) const = 0; + /// Return object information (absolute_path, bytes_size, ...) for metadata path. + /// object_storage_path is absolute. + virtual StoredObjects getStorageObjects(const std::string & path) const = 0; + + virtual std::string getObjectStorageRootPath() const = 0; }; using MetadataStoragePtr = std::shared_ptr; diff --git a/src/Disks/ObjectStorages/IObjectStorage.cpp b/src/Disks/ObjectStorages/IObjectStorage.cpp index d29ecc24aeb..f9c5c139b95 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.cpp +++ b/src/Disks/ObjectStorages/IObjectStorage.cpp @@ -6,6 +6,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + AsynchronousReaderPtr IObjectStorage::getThreadPoolReader() { constexpr size_t pool_size = 50; @@ -22,21 +27,11 @@ ThreadPool & IObjectStorage::getThreadPoolWriter() return writer; } -std::string IObjectStorage::getCacheBasePath() const -{ - return cache ? cache->getBasePath() : ""; -} - -void IObjectStorage::removeFromCache(const std::string & path) -{ - if (cache) - { - auto key = cache->hash(path); - cache->removeIfExists(key); - } -} - -void IObjectStorage::copyObjectToAnotherObjectStorage(const std::string & object_from, const std::string & object_to, IObjectStorage & object_storage_to, std::optional object_to_attributes) // NOLINT +void IObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT + const StoredObject & object_from, + const StoredObject & object_to, + IObjectStorage & object_storage_to, + std::optional object_to_attributes) { if (&object_storage_to == this) copyObject(object_from, object_to, object_to_attributes); @@ -47,4 +42,32 @@ void IObjectStorage::copyObjectToAnotherObjectStorage(const std::string & object out->finalize(); } +std::string IObjectStorage::getCacheBasePath() const +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getCacheBasePath() is not implemented for {}", getName()); +} + +void IObjectStorage::applyRemoteThrottlingSettings(ContextPtr context) +{ + std::unique_lock lock{throttlers_mutex}; + remote_read_throttler = context->getRemoteReadThrottler(); + remote_write_throttler = context->getRemoteWriteThrottler(); +} + +ReadSettings IObjectStorage::patchSettings(const ReadSettings & read_settings) const +{ + std::unique_lock lock{throttlers_mutex}; + ReadSettings settings{read_settings}; + settings.remote_throttler = remote_read_throttler; + return settings; +} + +WriteSettings IObjectStorage::patchSettings(const WriteSettings & write_settings) const +{ + std::unique_lock lock{throttlers_mutex}; + WriteSettings settings{write_settings}; + settings.remote_throttler = remote_write_throttler; + return settings; +} + } diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index 7532f2a3267..06398b11aec 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -12,8 +13,8 @@ #include #include +#include #include -#include #include @@ -25,23 +26,20 @@ class WriteBufferFromFileBase; using ObjectAttributes = std::map; -/// Path to a file (always absolute) and its size. -struct PathWithSize +struct RelativePathWithSize { - std::string path; - uint64_t bytes_size; + String relative_path; + size_t bytes_size; - PathWithSize() = default; + RelativePathWithSize() = default; - PathWithSize(const std::string & path_, uint64_t bytes_size_) - : path(path_) - , bytes_size(bytes_size_) - {} + RelativePathWithSize(const String & relative_path_, size_t bytes_size_) + : relative_path(relative_path_), bytes_size(bytes_size_) {} }; +using RelativePathsWithSize = std::vector; -/// List of paths with their sizes -using PathsWithSize = std::vector; -using RelativePathsWithSize = PathsWithSize; + +using StoredObjects = std::vector; struct ObjectMetadata { @@ -58,12 +56,12 @@ using FinalizeCallback = std::function; class IObjectStorage { public: - explicit IObjectStorage(FileCachePtr && cache_) - : cache(std::move(cache_)) - {} + IObjectStorage() = default; - /// Path exists or not - virtual bool exists(const std::string & path) const = 0; + virtual std::string getName() const = 0; + + /// Object exists or not + virtual bool exists(const StoredObject & object) const = 0; /// List on prefix, return children (relative paths) with their sizes. virtual void listPrefix(const std::string & path, RelativePathsWithSize & children) const = 0; @@ -72,61 +70,63 @@ public: /// at least size of object virtual ObjectMetadata getObjectMetadata(const std::string & path) const = 0; - /// Read single path from object storage - virtual std::unique_ptr readObject( /// NOLINT - const std::string & path, + /// Read single object + virtual std::unique_ptr readObject( /// NOLINT + const StoredObject & object, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const = 0; /// Read multiple objects with common prefix virtual std::unique_ptr readObjects( /// NOLINT - const PathsWithSize & paths_to_read, + const StoredObjects & objects, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const = 0; /// Open the file for write and return WriteBufferFromFileBase object. virtual std::unique_ptr writeObject( /// NOLINT - const std::string & path, + const StoredObject & object, WriteMode mode, std::optional attributes = {}, FinalizeCallback && finalize_callback = {}, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) = 0; + virtual bool isRemote() const = 0; + /// Remove object. Throws exception if object doesn't exists. - virtual void removeObject(const std::string & path) = 0; + virtual void removeObject(const StoredObject & object) = 0; /// Remove multiple objects. Some object storages can do batch remove in a more /// optimal way. - virtual void removeObjects(const PathsWithSize & paths) = 0; + virtual void removeObjects(const StoredObjects & objects) = 0; /// Remove object on path if exists - virtual void removeObjectIfExists(const std::string & path) = 0; + virtual void removeObjectIfExists(const StoredObject & object) = 0; /// Remove objects on path if exists - virtual void removeObjectsIfExist(const PathsWithSize & paths) = 0; + virtual void removeObjectsIfExist(const StoredObjects & object) = 0; /// Copy object with different attributes if required virtual void copyObject( /// NOLINT - const std::string & object_from, - const std::string & object_to, + const StoredObject & object_from, + const StoredObject & object_to, std::optional object_to_attributes = {}) = 0; /// Copy object to another instance of object storage /// by default just read the object from source object storage and write /// to destination through buffers. virtual void copyObjectToAnotherObjectStorage( /// NOLINT - const std::string & object_from, - const std::string & object_to, + const StoredObject & object_from, + const StoredObject & object_to, IObjectStorage & object_storage_to, std::optional object_to_attributes = {}); virtual ~IObjectStorage() = default; /// Path to directory with objects cache - std::string getCacheBasePath() const; + virtual std::string getCacheBasePath() const; static AsynchronousReaderPtr getThreadPoolReader(); @@ -136,8 +136,6 @@ public: virtual void startup() = 0; - void removeFromCache(const std::string & path); - /// Apply new settings, in most cases reiniatilize client and some other staff virtual void applyNewSettings( const Poco::Util::AbstractConfiguration & config, @@ -155,10 +153,35 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) = 0; + /// Generate blob name for passed absolute local path. + /// Path can be generated either independently or based on `path`. + virtual std::string generateBlobNameForPath(const std::string & path) = 0; + + /// Get unique id for passed absolute path in object storage. + virtual std::string getUniqueId(const std::string & path) const { return path; } + + virtual bool supportsAppend() const { return false; } + + /// Remove filesystem cache. `path` is a result of object.getPathKeyForCache() method, + /// which is used to define a cache key for the source object path. + virtual void removeCacheIfExists(const std::string & /* path */) {} + + virtual bool supportsCache() const { return false; } + protected: - FileCachePtr cache; + /// Should be called from implementation of applyNewSettings() + void applyRemoteThrottlingSettings(ContextPtr context); + + /// Should be used by implementation of read* and write* methods + ReadSettings patchSettings(const ReadSettings & read_settings) const; + WriteSettings patchSettings(const WriteSettings & write_settings) const; + +private: + mutable std::mutex throttlers_mutex; + ThrottlerPtr remote_read_throttler; + ThrottlerPtr remote_write_throttler; }; -using ObjectStoragePtr = std::unique_ptr; +using ObjectStoragePtr = std::shared_ptr; } diff --git a/src/Disks/ObjectStorages/IObjectStorage_fwd.h b/src/Disks/ObjectStorages/IObjectStorage_fwd.h new file mode 100644 index 00000000000..f6ebc883682 --- /dev/null +++ b/src/Disks/ObjectStorages/IObjectStorage_fwd.h @@ -0,0 +1,13 @@ +#pragma once +#include + +namespace DB +{ + +class IObjectStorage; +using ObjectStoragePtr = std::shared_ptr; + +class IMetadataStorage; +using MetadataStoragePtr = std::shared_ptr; + +} diff --git a/src/Disks/ObjectStorages/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/LocalObjectStorage.cpp new file mode 100644 index 00000000000..c052f2f0d77 --- /dev/null +++ b/src/Disks/ObjectStorages/LocalObjectStorage.cpp @@ -0,0 +1,173 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; +} + +namespace ErrorCodes +{ + extern const int CANNOT_UNLINK; +} + +LocalObjectStorage::LocalObjectStorage() + : log(&Poco::Logger::get("LocalObjectStorage")) +{ +} + +bool LocalObjectStorage::exists(const StoredObject & object) const +{ + return fs::exists(object.absolute_path); +} + +std::unique_ptr LocalObjectStorage::readObjects( /// NOLINT + const StoredObjects & objects, + const ReadSettings & read_settings, + std::optional read_hint, + std::optional file_size) const +{ + if (objects.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "LocalObjectStorage support read only from single object"); + + return readObject(objects[0], read_settings, read_hint, file_size); +} + +std::string LocalObjectStorage::getUniqueId(const std::string & path) const +{ + return toString(getINodeNumberFromPath(path)); +} + +std::unique_ptr LocalObjectStorage::readObject( /// NOLINT + const StoredObject & object, + const ReadSettings & read_settings, + std::optional read_hint, + std::optional file_size) const +{ + const auto & path = object.absolute_path; + + if (!file_size) + file_size = tryGetSizeFromFilePath(path); + + /// For now we cannot allow asynchronous reader from local filesystem when CachedObjectStorage is used. + ReadSettings modified_settings{read_settings}; + switch (modified_settings.local_fs_method) + { + case LocalFSReadMethod::pread_threadpool: + case LocalFSReadMethod::pread_fake_async: + { + modified_settings.local_fs_method = LocalFSReadMethod::pread; + LOG_INFO(log, "Changing local filesystem read method to `pread`"); + break; + } + default: + { + break; + } + } + + LOG_TEST(log, "Read object: {}", path); + return createReadBufferFromFileBase(path, modified_settings, read_hint, file_size); +} + +std::unique_ptr LocalObjectStorage::writeObject( /// NOLINT + const StoredObject & object, + WriteMode mode, + std::optional /* attributes */, + FinalizeCallback && /* finalize_callback */, + size_t buf_size, + const WriteSettings & /* write_settings */) +{ + const auto & path = object.absolute_path; + int flags = (mode == WriteMode::Append) ? (O_APPEND | O_CREAT | O_WRONLY) : -1; + LOG_TEST(log, "Write object: {}", path); + return std::make_unique(path, buf_size, flags); +} + +void LocalObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const +{ + fs::directory_iterator end_it; + for (auto it = fs::directory_iterator(path); it != end_it; ++it) + children.emplace_back(it->path().filename(), it->file_size()); +} + +void LocalObjectStorage::removeObject(const StoredObject & object) +{ + if (0 != unlink(object.absolute_path.data())) + throwFromErrnoWithPath("Cannot unlink file " + object.absolute_path, object.absolute_path, ErrorCodes::CANNOT_UNLINK); +} + +void LocalObjectStorage::removeObjects(const StoredObjects & objects) +{ + for (const auto & object : objects) + removeObject(object); +} + +void LocalObjectStorage::removeObjectIfExists(const StoredObject & object) +{ + if (exists(object)) + removeObject(object); +} + +void LocalObjectStorage::removeObjectsIfExist(const StoredObjects & objects) +{ + for (const auto & object : objects) + removeObjectIfExists(object); +} + +ObjectMetadata LocalObjectStorage::getObjectMetadata(const std::string & /* path */) const +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Metadata is not supported for LocalObjectStorage"); +} + +void LocalObjectStorage::copyObject( // NOLINT + const StoredObject & object_from, const StoredObject & object_to, std::optional /* object_to_attributes */) +{ + fs::path to = object_to.absolute_path; + fs::path from = object_from.absolute_path; + + /// Same logic as in DiskLocal. + if (object_from.absolute_path.ends_with('/')) + from = from.parent_path(); + if (fs::is_directory(from)) + to /= from.filename(); + + fs::copy(from, to, fs::copy_options::recursive | fs::copy_options::overwrite_existing); +} + +void LocalObjectStorage::shutdown() +{ +} + +void LocalObjectStorage::startup() +{ +} + +std::unique_ptr LocalObjectStorage::cloneObjectStorage( + const std::string & /* new_namespace */, + const Poco::Util::AbstractConfiguration & /* config */, + const std::string & /* config_prefix */, ContextPtr /* context */) +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "cloneObjectStorage() is not implemented for LocalObjectStorage"); +} + +void LocalObjectStorage::applyNewSettings( + const Poco::Util::AbstractConfiguration & /* config */, const std::string & /* config_prefix */, ContextPtr /* context */) +{ +} + +} diff --git a/src/Disks/ObjectStorages/LocalObjectStorage.h b/src/Disks/ObjectStorages/LocalObjectStorage.h new file mode 100644 index 00000000000..2225d0c72b7 --- /dev/null +++ b/src/Disks/ObjectStorages/LocalObjectStorage.h @@ -0,0 +1,91 @@ +#pragma once + +#include + +#include + +namespace Poco +{ +class Logger; +} + +namespace DB +{ + +class LocalObjectStorage : public IObjectStorage +{ +public: + LocalObjectStorage(); + + std::string getName() const override { return "LocalObjectStorage"; } + + bool exists(const StoredObject & object) const override; + + std::unique_ptr readObject( /// NOLINT + const StoredObject & object, + const ReadSettings & read_settings = ReadSettings{}, + std::optional read_hint = {}, + std::optional file_size = {}) const override; + + std::unique_ptr readObjects( /// NOLINT + const StoredObjects & objects, + const ReadSettings & read_settings = ReadSettings{}, + std::optional read_hint = {}, + std::optional file_size = {}) const override; + + /// Open the file for write and return WriteBufferFromFileBase object. + std::unique_ptr writeObject( /// NOLINT + const StoredObject & object, + WriteMode mode, + std::optional attributes = {}, + FinalizeCallback && finalize_callback = {}, + size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, + const WriteSettings & write_settings = {}) override; + + void listPrefix(const std::string & path, RelativePathsWithSize & children) const override; + + void removeObject(const StoredObject & object) override; + + void removeObjects(const StoredObjects & objects) override; + + void removeObjectIfExists(const StoredObject & object) override; + + void removeObjectsIfExist(const StoredObjects & objects) override; + + ObjectMetadata getObjectMetadata(const std::string & path) const override; + + void copyObject( /// NOLINT + const StoredObject & object_from, + const StoredObject & object_to, + std::optional object_to_attributes = {}) override; + + void shutdown() override; + + void startup() override; + + void applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; + + String getObjectsNamespace() const override { return ""; } + + std::unique_ptr cloneObjectStorage( + const std::string & new_namespace, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; + + bool supportsAppend() const override { return true; } + + std::string generateBlobNameForPath(const std::string & path) override { return path; } + + std::string getUniqueId(const std::string & path) const override; + + bool isRemote() const override { return false; } + +private: + Poco::Logger * log; +}; + +} diff --git a/src/Disks/ObjectStorages/MetadataFromDiskTransactionState.cpp b/src/Disks/ObjectStorages/MetadataFromDiskTransactionState.cpp new file mode 100644 index 00000000000..1ee87537114 --- /dev/null +++ b/src/Disks/ObjectStorages/MetadataFromDiskTransactionState.cpp @@ -0,0 +1,22 @@ +#include + +namespace DB +{ + +std::string toString(MetadataFromDiskTransactionState state) +{ + switch (state) + { + case MetadataFromDiskTransactionState::PREPARING: + return "PREPARING"; + case MetadataFromDiskTransactionState::FAILED: + return "FAILED"; + case MetadataFromDiskTransactionState::COMMITTED: + return "COMMITTED"; + case MetadataFromDiskTransactionState::PARTIALLY_ROLLED_BACK: + return "PARTIALLY_ROLLED_BACK"; + } + __builtin_unreachable(); +} + +} diff --git a/src/Disks/ObjectStorages/MetadataFromDiskTransactionState.h b/src/Disks/ObjectStorages/MetadataFromDiskTransactionState.h new file mode 100644 index 00000000000..3dc4c610e3a --- /dev/null +++ b/src/Disks/ObjectStorages/MetadataFromDiskTransactionState.h @@ -0,0 +1,17 @@ +#pragma once +#include + +namespace DB +{ + +enum class MetadataFromDiskTransactionState +{ + PREPARING, + FAILED, + COMMITTED, + PARTIALLY_ROLLED_BACK, +}; + +std::string toString(MetadataFromDiskTransactionState state); + +} diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp index 22502a6e2d1..a664433a3d0 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp @@ -1,11 +1,12 @@ #include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include namespace DB { @@ -15,36 +16,154 @@ namespace ErrorCodes extern const int FS_METADATA_ERROR; } - -std::string toString(MetadataFromDiskTransactionState state) +MetadataStorageFromDisk::MetadataStorageFromDisk(DiskPtr disk_, const std::string & object_storage_root_path_) + : disk(disk_) + , object_storage_root_path(object_storage_root_path_) { - switch (state) +} + +const std::string & MetadataStorageFromDisk::getPath() const +{ + return disk->getPath(); +} + +bool MetadataStorageFromDisk::exists(const std::string & path) const +{ + return disk->exists(path); +} + +bool MetadataStorageFromDisk::isFile(const std::string & path) const +{ + return disk->isFile(path); +} + +bool MetadataStorageFromDisk::isDirectory(const std::string & path) const +{ + return disk->isDirectory(path); +} + +Poco::Timestamp MetadataStorageFromDisk::getLastModified(const std::string & path) const +{ + return disk->getLastModified(path); +} + +time_t MetadataStorageFromDisk::getLastChanged(const std::string & path) const +{ + return disk->getLastChanged(path); +} + +uint64_t MetadataStorageFromDisk::getFileSize(const String & path) const +{ + auto metadata = readMetadata(path); + return metadata->getTotalSizeBytes(); +} + +std::vector MetadataStorageFromDisk::listDirectory(const std::string & path) const +{ + std::vector result_files; + disk->listFiles(path, result_files); + return result_files; +} + +DirectoryIteratorPtr MetadataStorageFromDisk::iterateDirectory(const std::string & path) const +{ + return disk->iterateDirectory(path); +} + + +std::string MetadataStorageFromDisk::readFileToString(const std::string & path) const +{ + auto buf = disk->readFile(path); + std::string result; + readStringUntilEOF(result, *buf); + return result; +} + +DiskObjectStorageMetadataPtr MetadataStorageFromDisk::readMetadataUnlocked(const std::string & path, std::shared_lock &) const +{ + auto metadata = std::make_unique(disk->getPath(), object_storage_root_path, path); + auto str = readFileToString(path); + metadata->deserializeFromString(str); + return metadata; +} + +DiskObjectStorageMetadataPtr MetadataStorageFromDisk::readMetadata(const std::string & path) const +{ + std::shared_lock lock(metadata_mutex); + return readMetadataUnlocked(path, lock); +} + +std::unordered_map MetadataStorageFromDisk::getSerializedMetadata(const std::vector & file_paths) const +{ + std::shared_lock lock(metadata_mutex); + std::unordered_map metadatas; + + for (const auto & path : file_paths) { - case MetadataFromDiskTransactionState::PREPARING: - return "PREPARING"; - case MetadataFromDiskTransactionState::FAILED: - return "FAILED"; - case MetadataFromDiskTransactionState::COMMITTED: - return "COMMITTED"; - case MetadataFromDiskTransactionState::PARTIALLY_ROLLED_BACK: - return "PARTIALLY_ROLLED_BACK"; + auto metadata = readMetadataUnlocked(path, lock); + metadata->resetRefCount(); + WriteBufferFromOwnString buf; + metadata->serialize(buf, false); + metadatas[path] = buf.str(); } - __builtin_unreachable(); + + return metadatas; } -void MetadataStorageFromDiskTransaction::writeStringToFile( /// NOLINT - const std::string & path, - const std::string & data) +void MetadataStorageFromDiskTransaction::createHardLink(const std::string & path_from, const std::string & path_to) { - addOperation(std::make_unique(path, *metadata_storage.disk, data)); + auto metadata = metadata_storage.readMetadata(path_from); + + metadata->incrementRefCount(); + + writeStringToFile(path_from, metadata->serializeToString()); + + addOperation(std::make_unique(path_from, path_to, *metadata_storage.getDisk())); } +MetadataTransactionPtr MetadataStorageFromDisk::createTransaction() const +{ + return std::make_shared(*this); +} + +StoredObjects MetadataStorageFromDisk::getStorageObjects(const std::string & path) const +{ + auto metadata = readMetadata(path); + + auto object_storage_relative_paths = metadata->getBlobsRelativePaths(); /// Relative paths. + + StoredObjects object_storage_paths; + object_storage_paths.reserve(object_storage_relative_paths.size()); + + /// Relative paths -> absolute. + for (auto & [object_relative_path, size] : object_storage_relative_paths) + { + auto object_path = fs::path(metadata->getBlobsCommonPrefix()) / object_relative_path; + StoredObject object{ object_path, size, [](const String & path_){ return path_; }}; + object_storage_paths.push_back(object); + } + + return object_storage_paths; +} + +uint32_t MetadataStorageFromDisk::getHardlinkCount(const std::string & path) const +{ + auto metadata = readMetadata(path); + return metadata->getRefCount(); +} + +const IMetadataStorage & MetadataStorageFromDiskTransaction::getStorageForNonTransactionalReads() const +{ + return metadata_storage; +} void MetadataStorageFromDiskTransaction::addOperation(MetadataOperationPtr && operation) { if (state != MetadataFromDiskTransactionState::PREPARING) - throw Exception(ErrorCodes::FS_METADATA_ERROR, "Cannot add operations to transaction in {} state, it should be in {} state", - toString(state), toString(MetadataFromDiskTransactionState::PREPARING)); + throw Exception( + ErrorCodes::FS_METADATA_ERROR, + "Cannot add operations to transaction in {} state, it should be in {} state", + toString(state), toString(MetadataFromDiskTransactionState::PREPARING)); operations.emplace_back(std::move(operation)); } @@ -52,8 +171,10 @@ void MetadataStorageFromDiskTransaction::addOperation(MetadataOperationPtr && op void MetadataStorageFromDiskTransaction::commit() { if (state != MetadataFromDiskTransactionState::PREPARING) - throw Exception(ErrorCodes::FS_METADATA_ERROR, "Cannot commit transaction in {} state, it should be in {} state", - toString(state), toString(MetadataFromDiskTransactionState::PREPARING)); + throw Exception( + ErrorCodes::FS_METADATA_ERROR, + "Cannot commit transaction in {} state, it should be in {} state", + toString(state), toString(MetadataFromDiskTransactionState::PREPARING)); { std::lock_guard lock(metadata_storage.metadata_mutex); @@ -65,6 +186,7 @@ void MetadataStorageFromDiskTransaction::commit() } catch (Exception & ex) { + tryLogCurrentException(__PRETTY_FUNCTION__); ex.addMessage(fmt::format("While committing metadata operation #{}", i)); state = MetadataFromDiskTransactionState::FAILED; rollback(i); @@ -114,124 +236,56 @@ void MetadataStorageFromDiskTransaction::rollback(size_t until_pos) } } -const std::string & MetadataStorageFromDisk::getPath() const +void MetadataStorageFromDiskTransaction::writeStringToFile( + const std::string & path, + const std::string & data) { - return disk->getPath(); -} - -bool MetadataStorageFromDisk::exists(const std::string & path) const -{ - return disk->exists(path); -} - -bool MetadataStorageFromDisk::isFile(const std::string & path) const -{ - return disk->isFile(path); -} - - -bool MetadataStorageFromDisk::isDirectory(const std::string & path) const -{ - return disk->isDirectory(path); -} - -Poco::Timestamp MetadataStorageFromDisk::getLastModified(const std::string & path) const -{ - return disk->getLastModified(path); -} - -time_t MetadataStorageFromDisk::getLastChanged(const std::string & path) const -{ - return disk->getLastChanged(path); -} - -uint64_t MetadataStorageFromDisk::getFileSize(const String & path) const -{ - auto metadata = readMetadata(path); - return metadata->getTotalSizeBytes(); -} - -std::vector MetadataStorageFromDisk::listDirectory(const std::string & path) const -{ - std::vector result_files; - disk->listFiles(path, result_files); - return result_files; -} - -DirectoryIteratorPtr MetadataStorageFromDisk::iterateDirectory(const std::string & path) const -{ - return disk->iterateDirectory(path); -} - - -std::string MetadataStorageFromDisk::readFileToString(const std::string & path) const -{ - auto buf = disk->readFile(path); - std::string result; - readStringUntilEOF(result, *buf); - return result; -} - -void MetadataStorageFromDiskTransaction::createEmptyMetadataFile(const std::string & path) -{ - auto metadata = std::make_unique(metadata_storage.disk->getPath(), metadata_storage.root_path_for_remote_metadata, path); - writeStringToFile(path, metadata->serializeToString()); + addOperation(std::make_unique(path, *metadata_storage.getDisk(), data)); } void MetadataStorageFromDiskTransaction::setLastModified(const std::string & path, const Poco::Timestamp & timestamp) { - addOperation(std::make_unique(path, timestamp, *metadata_storage.disk)); + addOperation(std::make_unique(path, timestamp, *metadata_storage.getDisk())); } void MetadataStorageFromDiskTransaction::unlinkFile(const std::string & path) { - addOperation(std::make_unique(path, *metadata_storage.disk)); + addOperation(std::make_unique(path, *metadata_storage.getDisk())); } void MetadataStorageFromDiskTransaction::removeRecursive(const std::string & path) { - addOperation(std::make_unique(path, *metadata_storage.disk)); + addOperation(std::make_unique(path, *metadata_storage.getDisk())); } void MetadataStorageFromDiskTransaction::createDirectory(const std::string & path) { - addOperation(std::make_unique(path, *metadata_storage.disk)); + addOperation(std::make_unique(path, *metadata_storage.getDisk())); } -void MetadataStorageFromDiskTransaction::createDicrectoryRecursive(const std::string & path) +void MetadataStorageFromDiskTransaction::createDirectoryRecursive(const std::string & path) { - addOperation(std::make_unique(path, *metadata_storage.disk)); + addOperation(std::make_unique(path, *metadata_storage.getDisk())); } void MetadataStorageFromDiskTransaction::removeDirectory(const std::string & path) { - addOperation(std::make_unique(path, *metadata_storage.disk)); -} - -void MetadataStorageFromDiskTransaction::createHardLink(const std::string & path_from, const std::string & path_to) -{ - auto metadata = metadata_storage.readMetadata(path_from); - - metadata->incrementRefCount(); - - writeStringToFile(path_from, metadata->serializeToString()); - - addOperation(std::make_unique(path_from, path_to, *metadata_storage.disk)); + addOperation(std::make_unique(path, *metadata_storage.getDisk())); } void MetadataStorageFromDiskTransaction::moveFile(const std::string & path_from, const std::string & path_to) { - addOperation(std::make_unique(path_from, path_to, *metadata_storage.disk)); + addOperation(std::make_unique(path_from, path_to, *metadata_storage.getDisk())); } void MetadataStorageFromDiskTransaction::moveDirectory(const std::string & path_from, const std::string & path_to) { - addOperation(std::make_unique(path_from, path_to, *metadata_storage.disk)); + addOperation(std::make_unique(path_from, path_to, *metadata_storage.getDisk())); } void MetadataStorageFromDiskTransaction::replaceFile(const std::string & path_from, const std::string & path_to) { - addOperation(std::make_unique(path_from, path_to, *metadata_storage.disk)); + addOperation(std::make_unique(path_from, path_to, *metadata_storage.getDisk())); } void MetadataStorageFromDiskTransaction::setReadOnly(const std::string & path) @@ -241,11 +295,23 @@ void MetadataStorageFromDiskTransaction::setReadOnly(const std::string & path) writeStringToFile(path, metadata->serializeToString()); } +void MetadataStorageFromDiskTransaction::createEmptyMetadataFile(const std::string & path) +{ + auto metadata = std::make_unique( + metadata_storage.getDisk()->getPath(), metadata_storage.getObjectStorageRootPath(), path); + writeStringToFile(path, metadata->serializeToString()); +} + void MetadataStorageFromDiskTransaction::createMetadataFile(const std::string & path, const std::string & blob_name, uint64_t size_in_bytes) { - DiskObjectStorageMetadataPtr metadata = std::make_unique(metadata_storage.disk->getPath(), metadata_storage.root_path_for_remote_metadata, path); + DiskObjectStorageMetadataPtr metadata = std::make_unique( + metadata_storage.getDisk()->getPath(), metadata_storage.getObjectStorageRootPath(), path); + metadata->addObject(blob_name, size_in_bytes); - writeStringToFile(path, metadata->serializeToString()); + + auto data = metadata->serializeToString(); + if (!data.empty()) + addOperation(std::make_unique(path, *metadata_storage.getDisk(), data)); } void MetadataStorageFromDiskTransaction::addBlobToMetadata(const std::string & path, const std::string & blob_name, uint64_t size_in_bytes) @@ -254,73 +320,13 @@ void MetadataStorageFromDiskTransaction::addBlobToMetadata(const std::string & p if (metadata_storage.exists(path)) { metadata = metadata_storage.readMetadata(path); + metadata->addObject(blob_name, size_in_bytes); + writeStringToFile(path, metadata->serializeToString()); } else { - metadata = std::make_unique(metadata_storage.disk->getPath(), metadata_storage.root_path_for_remote_metadata, path); + createMetadataFile(path, blob_name, size_in_bytes); } - - metadata->addObject(blob_name, size_in_bytes); - writeStringToFile(path, metadata->serializeToString()); -} - -DiskObjectStorageMetadataPtr MetadataStorageFromDisk::readMetadataUnlocked(const std::string & path, std::shared_lock &) const -{ - auto metadata = std::make_unique(disk->getPath(), root_path_for_remote_metadata, path); - auto str = readFileToString(path); - metadata->deserializeFromString(str); - return metadata; -} - -DiskObjectStorageMetadataPtr MetadataStorageFromDisk::readMetadata(const std::string & path) const -{ - std::shared_lock lock(metadata_mutex); - return readMetadataUnlocked(path, lock); -} - -std::unordered_map MetadataStorageFromDisk::getSerializedMetadata(const std::vector & file_paths) const -{ - std::shared_lock lock(metadata_mutex); - std::unordered_map metadatas; - - for (const auto & path : file_paths) - { - auto metadata = readMetadataUnlocked(path, lock); - metadata->resetRefCount(); - WriteBufferFromOwnString buf; - metadata->serialize(buf, false); - metadatas[path] = buf.str(); - } - - return metadatas; -} - -MetadataTransactionPtr MetadataStorageFromDisk::createTransaction() const -{ - return std::make_shared(*this); -} - -PathsWithSize MetadataStorageFromDisk::getObjectStoragePaths(const std::string & path) const -{ - auto metadata = readMetadata(path); - - auto object_storage_relative_paths = metadata->getBlobsRelativePaths(); /// Relative paths. - fs::path root_path = metadata->getBlobsCommonPrefix(); - - PathsWithSize object_storage_paths; - object_storage_paths.reserve(object_storage_relative_paths.size()); - - /// Relative paths -> absolute. - for (auto & [object_relative_path, size] : object_storage_relative_paths) - object_storage_paths.emplace_back(root_path / object_relative_path, size); - - return object_storage_paths; -} - -uint32_t MetadataStorageFromDisk::getHardlinkCount(const std::string & path) const -{ - auto metadata = readMetadata(path); - return metadata->getRefCount(); } void MetadataStorageFromDiskTransaction::unlinkMetadata(const std::string & path) diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDisk.h b/src/Disks/ObjectStorages/MetadataStorageFromDisk.h index 8dd47d18fdb..e84b8fe36eb 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDisk.h +++ b/src/Disks/ObjectStorages/MetadataStorageFromDisk.h @@ -4,36 +4,24 @@ #include #include -#include "MetadataStorageFromDiskTransactionOperations.h" +#include +#include namespace DB { -enum class MetadataFromDiskTransactionState -{ - PREPARING, - FAILED, - COMMITTED, - PARTIALLY_ROLLED_BACK, -}; - -std::string toString(MetadataFromDiskTransactionState state); - class MetadataStorageFromDisk final : public IMetadataStorage { private: - friend struct MetadataStorageFromDiskTransaction; + friend class MetadataStorageFromDiskTransaction; - DiskPtr disk; - std::string root_path_for_remote_metadata; mutable std::shared_mutex metadata_mutex; + DiskPtr disk; + std::string object_storage_root_path; + public: - MetadataStorageFromDisk(DiskPtr disk_, const std::string & root_path_from_remote_metadata_) - : disk(disk_) - , root_path_for_remote_metadata(root_path_from_remote_metadata_) - { - } + MetadataStorageFromDisk(DiskPtr disk_, const std::string & object_storage_root_path_); MetadataTransactionPtr createTransaction() const override; @@ -59,17 +47,21 @@ public: std::unordered_map getSerializedMetadata(const std::vector & file_paths) const override; - PathsWithSize getObjectStoragePaths(const std::string & path) const override; - uint32_t getHardlinkCount(const std::string & path) const override; + DiskPtr getDisk() const { return disk; } + + StoredObjects getStorageObjects(const std::string & path) const override; + + std::string getObjectStorageRootPath() const override { return object_storage_root_path; } private: DiskObjectStorageMetadataPtr readMetadata(const std::string & path) const; + DiskObjectStorageMetadataPtr readMetadataUnlocked(const std::string & path, std::shared_lock & lock) const; }; -struct MetadataStorageFromDiskTransaction final : public IMetadataTransaction +class MetadataStorageFromDiskTransaction final : public IMetadataTransaction { private: const MetadataStorageFromDisk & metadata_storage; @@ -78,6 +70,7 @@ private: MetadataFromDiskTransactionState state{MetadataFromDiskTransactionState::PREPARING}; void addOperation(MetadataOperationPtr && operation); + void rollback(size_t until_pos); public: @@ -85,12 +78,11 @@ public: : metadata_storage(metadata_storage_) {} - const IMetadataStorage & getStorageForNonTransactionalReads() const override - { - return metadata_storage; - } + ~MetadataStorageFromDiskTransaction() override = default; - void commit() override; + const IMetadataStorage & getStorageForNonTransactionalReads() const final; + + void commit() final; void writeStringToFile(const std::string & path, const std::string & data) override; @@ -108,7 +100,7 @@ public: void createDirectory(const std::string & path) override; - void createDicrectoryRecursive(const std::string & path) override; + void createDirectoryRecursive(const std::string & path) override; void removeDirectory(const std::string & path) override; @@ -123,8 +115,6 @@ public: void replaceFile(const std::string & path_from, const std::string & path_to) override; void unlinkMetadata(const std::string & path) override; - - ~MetadataStorageFromDiskTransaction() override = default; }; diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 8a0402ef1b2..55c466d45f6 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -26,6 +26,8 @@ #include #include +#include +#include namespace DB { @@ -84,6 +86,11 @@ void logIfError(const Aws::Utils::Outcome & response, std::functi } +std::string S3ObjectStorage::generateBlobNameForPath(const std::string & /* path */) +{ + return getRandomASCIIString(); +} + Aws::S3::Model::HeadObjectOutcome S3ObjectStorage::requestObjectHeadData(const std::string & bucket_from, const std::string & key) const { auto client_ptr = client.get(); @@ -94,9 +101,9 @@ Aws::S3::Model::HeadObjectOutcome S3ObjectStorage::requestObjectHeadData(const s return client_ptr->HeadObject(request); } -bool S3ObjectStorage::exists(const std::string & path) const +bool S3ObjectStorage::exists(const StoredObject & object) const { - auto object_head = requestObjectHeadData(bucket, path); + auto object_head = requestObjectHeadData(bucket, object.absolute_path); if (!object_head.IsSuccess()) { if (object_head.GetError().GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND) @@ -107,22 +114,32 @@ bool S3ObjectStorage::exists(const std::string & path) const return true; } +String S3ObjectStorage::getCacheBasePath() const +{ + if (!cache) + return ""; + + return cache->getBasePath(); +} + +void S3ObjectStorage::removeCacheIfExists(const std::string & path_key) +{ + if (!cache || path_key.empty()) + return; + + IFileCache::Key key = cache->hash(path_key); + cache->removeIfExists(key); +} std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT - const PathsWithSize & paths_to_read, + const StoredObjects & objects, const ReadSettings & read_settings, std::optional, std::optional) const { + assert(!objects[0].getPathKeyForCache().empty()); - ReadSettings disk_read_settings{read_settings}; - if (cache) - { - if (IFileCache::isReadOnly()) - disk_read_settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = true; - - disk_read_settings.remote_fs_cache = cache; - } + ReadSettings disk_read_settings = patchSettings(read_settings); auto settings_ptr = s3_settings.get(); @@ -130,7 +147,7 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT client.get(), bucket, version_id, - paths_to_read, + objects, settings_ptr->s3_settings.max_single_read_retries, disk_read_settings); @@ -146,53 +163,54 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT } } -std::unique_ptr S3ObjectStorage::readObject( /// NOLINT - const std::string & path, +std::unique_ptr S3ObjectStorage::readObject( /// NOLINT + const StoredObject & object, const ReadSettings & read_settings, std::optional, std::optional) const { auto settings_ptr = s3_settings.get(); - ReadSettings disk_read_settings{read_settings}; - if (cache) - { - if (IFileCache::isReadOnly()) - disk_read_settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = true; - - disk_read_settings.remote_fs_cache = cache; - } - - return std::make_unique(client.get(), bucket, path, version_id, settings_ptr->s3_settings.max_single_read_retries, disk_read_settings); + return std::make_unique( + client.get(), + bucket, + object.absolute_path, + version_id, + settings_ptr->s3_settings.max_single_read_retries, + patchSettings(read_settings)); } - std::unique_ptr S3ObjectStorage::writeObject( /// NOLINT - const std::string & path, + const StoredObject & object, WriteMode mode, // S3 doesn't support append, only rewrite std::optional attributes, FinalizeCallback && finalize_callback, size_t buf_size, const WriteSettings & write_settings) { + WriteSettings disk_write_settings = IObjectStorage::patchSettings(write_settings); + if (mode != WriteMode::Rewrite) throw Exception(ErrorCodes::BAD_ARGUMENTS, "S3 doesn't support append to files"); bool cache_on_write = cache - && fs::path(path).extension() != ".tmp" - && write_settings.enable_filesystem_cache_on_write_operations - && FileCacheFactory::instance().getSettings(getCacheBasePath()).cache_on_write_operations; + && write_settings.enable_filesystem_cache_on_write_operations + && FileCacheFactory::instance().getSettings(getCacheBasePath()).cache_on_write_operations; auto settings_ptr = s3_settings.get(); auto s3_buffer = std::make_unique( client.get(), bucket, - path, + object.absolute_path, settings_ptr->s3_settings, attributes, - buf_size, threadPoolCallbackRunner(getThreadPoolWriter()), + buf_size, + threadPoolCallbackRunner(getThreadPoolWriter()), + disk_write_settings, cache_on_write ? cache : nullptr); - return std::make_unique(std::move(s3_buffer), std::move(finalize_callback), path); + + return std::make_unique( + std::move(s3_buffer), std::move(finalize_callback), object.absolute_path); } void S3ObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize & children) const @@ -224,27 +242,27 @@ void S3ObjectStorage::listPrefix(const std::string & path, RelativePathsWithSize } while (outcome.GetResult().GetIsTruncated()); } -void S3ObjectStorage::removeObjectImpl(const std::string & path, bool if_exists) +void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exists) { auto client_ptr = client.get(); Aws::S3::Model::DeleteObjectRequest request; request.SetBucket(bucket); - request.SetKey(path); + request.SetKey(object.absolute_path); auto outcome = client_ptr->DeleteObject(request); throwIfUnexpectedError(outcome, if_exists); } -void S3ObjectStorage::removeObjectsImpl(const PathsWithSize & paths, bool if_exists) +void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_exists) { - if (paths.empty()) + if (objects.empty()) return; if (!s3_capabilities.support_batch_delete) { - for (const auto & [path, _] : paths) - removeObjectImpl(path, if_exists); + for (const auto & object : objects) + removeObjectImpl(object, if_exists); } else { @@ -254,19 +272,19 @@ void S3ObjectStorage::removeObjectsImpl(const PathsWithSize & paths, bool if_exi size_t chunk_size_limit = settings_ptr->objects_chunk_size_to_delete; size_t current_position = 0; - while (current_position < paths.size()) + while (current_position < objects.size()) { std::vector current_chunk; String keys; - for (; current_position < paths.size() && current_chunk.size() < chunk_size_limit; ++current_position) + for (; current_position < objects.size() && current_chunk.size() < chunk_size_limit; ++current_position) { Aws::S3::Model::ObjectIdentifier obj; - obj.SetKey(paths[current_position].path); + obj.SetKey(objects[current_position].absolute_path); current_chunk.push_back(obj); if (!keys.empty()) keys += ", "; - keys += paths[current_position].path; + keys += objects[current_position].absolute_path; } Aws::S3::Model::Delete delkeys; @@ -281,24 +299,24 @@ void S3ObjectStorage::removeObjectsImpl(const PathsWithSize & paths, bool if_exi } } -void S3ObjectStorage::removeObject(const std::string & path) +void S3ObjectStorage::removeObject(const StoredObject & object) { - removeObjectImpl(path, false); + removeObjectImpl(object, false); } -void S3ObjectStorage::removeObjectIfExists(const std::string & path) +void S3ObjectStorage::removeObjectIfExists(const StoredObject & object) { - removeObjectImpl(path, true); + removeObjectImpl(object, true); } -void S3ObjectStorage::removeObjects(const PathsWithSize & paths) +void S3ObjectStorage::removeObjects(const StoredObjects & objects) { - removeObjectsImpl(paths, false); + removeObjectsImpl(objects, false); } -void S3ObjectStorage::removeObjectsIfExist(const PathsWithSize & paths) +void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects) { - removeObjectsImpl(paths, true); + removeObjectsImpl(objects, true); } ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) const @@ -316,16 +334,28 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) cons return result; } -void S3ObjectStorage::copyObjectToAnotherObjectStorage(const std::string & object_from, const std::string & object_to, IObjectStorage & object_storage_to, std::optional object_to_attributes) // NOLINT +void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT + const StoredObject & object_from, + const StoredObject & object_to, + IObjectStorage & object_storage_to, + std::optional object_to_attributes) { /// Shortcut for S3 if (auto * dest_s3 = dynamic_cast(&object_storage_to); dest_s3 != nullptr) - copyObjectImpl(bucket, object_from, dest_s3->bucket, object_to, {}, object_to_attributes); + { + copyObjectImpl(bucket, object_from.absolute_path, dest_s3->bucket, object_to.absolute_path, {}, object_to_attributes); + } else + { IObjectStorage::copyObjectToAnotherObjectStorage(object_from, object_to, object_storage_to, object_to_attributes); + } } -void S3ObjectStorage::copyObjectImpl(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, +void S3ObjectStorage::copyObjectImpl( + const String & src_bucket, + const String & src_key, + const String & dst_bucket, + const String & dst_key, std::optional head, std::optional metadata) const { @@ -351,7 +381,11 @@ void S3ObjectStorage::copyObjectImpl(const String & src_bucket, const String & s throwIfError(outcome); } -void S3ObjectStorage::copyObjectMultipartImpl(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, +void S3ObjectStorage::copyObjectMultipartImpl( + const String & src_bucket, + const String & src_key, + const String & dst_bucket, + const String & dst_key, std::optional head, std::optional metadata) const { @@ -428,13 +462,35 @@ void S3ObjectStorage::copyObjectMultipartImpl(const String & src_bucket, const S } } -void S3ObjectStorage::copyObject(const std::string & object_from, const std::string & object_to, std::optional object_to_attributes) // NOLINT +void S3ObjectStorage::copyObject( // NOLINT + const StoredObject & object_from, const StoredObject & object_to, std::optional object_to_attributes) { - auto head = requestObjectHeadData(bucket, object_from).GetResult(); - if (head.GetContentLength() >= static_cast(5UL * 1024 * 1024 * 1024)) - copyObjectMultipartImpl(bucket, object_from, bucket, object_to, head, object_to_attributes); + auto head = requestObjectHeadData(bucket, object_from.absolute_path).GetResult(); + static constexpr int64_t multipart_upload_threashold = 5UL * 1024 * 1024 * 1024; + + if (head.GetContentLength() >= multipart_upload_threashold) + { + copyObjectMultipartImpl( + bucket, object_from.absolute_path, bucket, object_to.absolute_path, head, object_to_attributes); + } else - copyObjectImpl(bucket, object_from, bucket, object_to, head, object_to_attributes); + { + copyObjectImpl( + bucket, object_from.absolute_path, bucket, object_to.absolute_path, head, object_to_attributes); + } +} + +ReadSettings S3ObjectStorage::patchSettings(const ReadSettings & read_settings) const +{ + ReadSettings settings{read_settings}; + if (cache) + { + if (IFileCache::isReadOnly()) + settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = true; + + settings.remote_fs_cache = cache; + } + return IObjectStorage::patchSettings(settings); } void S3ObjectStorage::setNewSettings(std::unique_ptr && s3_settings_) @@ -469,14 +525,16 @@ void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & { s3_settings.set(getSettings(config, config_prefix, context)); client.set(getClient(config, config_prefix, context)); + applyRemoteThrottlingSettings(context); } -std::unique_ptr S3ObjectStorage::cloneObjectStorage(const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) +std::unique_ptr S3ObjectStorage::cloneObjectStorage( + const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) { return std::make_unique( - nullptr, getClient(config, config_prefix, context), + getClient(config, config_prefix, context), getSettings(config, config_prefix, context), - version_id, s3_capabilities, new_namespace); + version_id, s3_capabilities, new_namespace, nullptr); } } diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index 86a8bd5d5b3..98397224629 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -43,37 +43,40 @@ class S3ObjectStorage : public IObjectStorage { public: S3ObjectStorage( - FileCachePtr && cache_, std::unique_ptr && client_, std::unique_ptr && s3_settings_, String version_id_, const S3Capabilities & s3_capabilities_, - String bucket_) - : IObjectStorage(std::move(cache_)) - , bucket(bucket_) + String bucket_, + FileCachePtr cache_) + : bucket(bucket_) , client(std::move(client_)) , s3_settings(std::move(s3_settings_)) , s3_capabilities(s3_capabilities_) , version_id(std::move(version_id_)) - {} + , cache(cache_) + { + } - bool exists(const std::string & path) const override; + std::string getName() const override { return "S3ObjectStorage"; } - std::unique_ptr readObject( /// NOLINT - const std::string & path, + bool exists(const StoredObject & object) const override; + + std::unique_ptr readObject( /// NOLINT + const StoredObject & object, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT - const PathsWithSize & paths_to_read, + const StoredObjects & objects, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; /// Open the file for write and return WriteBufferFromFileBase object. std::unique_ptr writeObject( /// NOLINT - const std::string & path, + const StoredObject & object, WriteMode mode, std::optional attributes = {}, FinalizeCallback && finalize_callback = {}, @@ -82,31 +85,30 @@ public: void listPrefix(const std::string & path, RelativePathsWithSize & children) const override; - /// Remove file. Throws exception if file doesn't exist or it's a directory. /// Uses `DeleteObjectRequest`. - void removeObject(const std::string & path) override; + void removeObject(const StoredObject & object) override; /// Uses `DeleteObjectsRequest` if it is allowed by `s3_capabilities`, otherwise `DeleteObjectRequest`. /// `DeleteObjectsRequest` is not supported on GCS, see https://issuetracker.google.com/issues/162653700 . - void removeObjects(const PathsWithSize & paths) override; + void removeObjects(const StoredObjects & objects) override; /// Uses `DeleteObjectRequest`. - void removeObjectIfExists(const std::string & path) override; + void removeObjectIfExists(const StoredObject & object) override; /// Uses `DeleteObjectsRequest` if it is allowed by `s3_capabilities`, otherwise `DeleteObjectRequest`. /// `DeleteObjectsRequest` does not exist on GCS, see https://issuetracker.google.com/issues/162653700 . - void removeObjectsIfExist(const PathsWithSize & paths) override; + void removeObjectsIfExist(const StoredObjects & objects) override; ObjectMetadata getObjectMetadata(const std::string & path) const override; void copyObject( /// NOLINT - const std::string & object_from, - const std::string & object_to, + const StoredObject & object_from, + const StoredObject & object_to, std::optional object_to_attributes = {}) override; void copyObjectToAnotherObjectStorage( /// NOLINT - const std::string & object_from, - const std::string & object_to, + const StoredObject & object_from, + const StoredObject & object_to, IObjectStorage & object_storage_to, std::optional object_to_attributes = {}) override; @@ -119,9 +121,13 @@ public: const std::string & config_prefix, ContextPtr context) override; - void setCapabilitiesSupportBatchDelete(bool value) { s3_capabilities.support_batch_delete = value; } + std::string getObjectsNamespace() const override { return bucket; } - String getObjectsNamespace() const override { return bucket; } + std::string generateBlobNameForPath(const std::string & path) override; + + bool isRemote() const override { return true; } + + void setCapabilitiesSupportBatchDelete(bool value) { s3_capabilities.support_batch_delete = value; } std::unique_ptr cloneObjectStorage( const std::string & new_namespace, @@ -129,7 +135,15 @@ public: const std::string & config_prefix, ContextPtr context) override; + bool supportsCache() const override { return true; } + + void removeCacheIfExists(const std::string & path_key) override; + + String getCacheBasePath() const override; + private: + ReadSettings patchSettings(const ReadSettings & read_settings) const; + void setNewSettings(std::unique_ptr && s3_settings_); void setNewClient(std::unique_ptr && client_); @@ -150,8 +164,8 @@ private: std::optional head = std::nullopt, std::optional metadata = std::nullopt) const; - void removeObjectImpl(const std::string & path, bool if_exists); - void removeObjectsImpl(const PathsWithSize & paths, bool if_exists); + void removeObjectImpl(const StoredObject & object, bool if_exists); + void removeObjectsImpl(const StoredObjects & objects, bool if_exists); Aws::S3::Model::HeadObjectOutcome requestObjectHeadData(const std::string & bucket_from, const std::string & key) const; @@ -162,6 +176,8 @@ private: S3Capabilities s3_capabilities; const String version_id; + + FileCachePtr cache; }; } diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 17e401a7428..11f7b2e8ad7 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -12,13 +12,16 @@ #include #include #include -#include + #include +#include #include #include #include +#include #include #include + #include namespace DB diff --git a/src/Disks/ObjectStorages/S3/diskSettings.h b/src/Disks/ObjectStorages/S3/diskSettings.h index b06e412b6e9..9092f5e712d 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.h +++ b/src/Disks/ObjectStorages/S3/diskSettings.h @@ -4,22 +4,22 @@ #if USE_AWS_S3 -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include + +namespace Aws +{ +namespace S3 +{ +class S3Client; +} +} namespace DB { +struct S3ObjectStorageSettings; + std::unique_ptr getSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context); std::unique_ptr getClient(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context); diff --git a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp index 8bd77d0fa7e..f946e9757c1 100644 --- a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp +++ b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp @@ -75,10 +75,10 @@ void checkRemoveAccess(IDisk & disk) bool checkBatchRemoveIsMissing(S3ObjectStorage & storage, const String & key_with_trailing_slash) { - const String path = key_with_trailing_slash + "_test_remove_objects_capability"; + StoredObject object(key_with_trailing_slash + "_test_remove_objects_capability"); try { - auto file = storage.writeObject(path, WriteMode::Rewrite); + auto file = storage.writeObject(object, WriteMode::Rewrite); file->write("test", 4); file->finalize(); } @@ -86,7 +86,7 @@ bool checkBatchRemoveIsMissing(S3ObjectStorage & storage, const String & key_wit { try { - storage.removeObject(path); + storage.removeObject(object); } catch (...) { @@ -96,14 +96,14 @@ bool checkBatchRemoveIsMissing(S3ObjectStorage & storage, const String & key_wit try { /// Uses `DeleteObjects` request (batch delete). - storage.removeObjects({{ path, 0 }}); + storage.removeObjects({object}); return false; } catch (const Exception &) { try { - storage.removeObject(path); + storage.removeObject(object); } catch (...) { @@ -137,9 +137,9 @@ void registerDiskS3(DiskFactory & factory) S3Capabilities s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); auto s3_storage = std::make_unique( - std::move(cache), getClient(config, config_prefix, context), + getClient(config, config_prefix, context), getSettings(config, config_prefix, context), - uri.version_id, s3_capabilities, uri.bucket); + uri.version_id, s3_capabilities, uri.bucket, cache); bool skip_access_check = config.getBool(config_prefix + ".skip_access_check", false); diff --git a/src/Disks/ObjectStorages/StoredObject.cpp b/src/Disks/ObjectStorages/StoredObject.cpp new file mode 100644 index 00000000000..1341f468bf2 --- /dev/null +++ b/src/Disks/ObjectStorages/StoredObject.cpp @@ -0,0 +1,55 @@ +#include + +#include +#include +#include + + +namespace DB +{ + +StoredObject::StoredObject( + const std::string & absolute_path_, + uint64_t bytes_size_, + PathKeyForCacheCreator && path_key_for_cache_creator_) + : absolute_path(absolute_path_) + , bytes_size(bytes_size_) + , path_key_for_cache_creator(std::move(path_key_for_cache_creator_)) +{ +} + +std::string StoredObject::getPathKeyForCache() const +{ + if (!path_key_for_cache_creator) + return ""; /// This empty result need to be used with care. + + return path_key_for_cache_creator(absolute_path); +} + +StoredObject StoredObject::create( + const IObjectStorage & object_storage, const std::string & object_path, size_t object_size, bool object_bypasses_cache) +{ + if (object_bypasses_cache) + return StoredObject(object_path, object_size, {}); + + auto path_key_for_cache_creator = [&object_storage](const std::string & path) -> String + { + try + { + return object_storage.getUniqueId(path); + } + catch (...) + { + LOG_DEBUG( + &Poco::Logger::get("StoredObject"), + "Object does not exist while getting cache path hint (object path: {})", + path); + + return ""; + } + }; + + return StoredObject(object_path, object_size, std::move(path_key_for_cache_creator)); +} + +} diff --git a/src/Disks/ObjectStorages/StoredObject.h b/src/Disks/ObjectStorages/StoredObject.h new file mode 100644 index 00000000000..0b8028b6ef6 --- /dev/null +++ b/src/Disks/ObjectStorages/StoredObject.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/// Object metadata: path, size, path_key_for_cache. +struct StoredObject +{ + std::string absolute_path; + + uint64_t bytes_size; + + std::string getPathKeyForCache() const; + + /// Create `StoredObject` based on metadata storage and blob name of the object. + static StoredObject create( + const IObjectStorage & object_storage, + const std::string & object_path, + size_t object_size = 0, + bool object_bypasses_cache = false); + + /// Optional hint for cache. Use delayed initialization + /// because somecache hint implementation requires it. + using PathKeyForCacheCreator = std::function; + PathKeyForCacheCreator path_key_for_cache_creator; + + explicit StoredObject( + const std::string & absolute_path_, + uint64_t bytes_size_ = 0, + PathKeyForCacheCreator && path_key_for_cache_creator_ = {}); +}; + +} diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 9006c9276d4..b9af9d61da0 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -77,18 +77,18 @@ namespace return convertChar(c1) == convertChar(c2); } - static bool equals(const std::string_view & s1, const std::string_view & s2) + static bool equals(std::string_view s1, std::string_view s2) { return (s1.length() == s2.length()) && std::equal(s1.begin(), s1.end(), s2.begin(), [](char c1, char c2) { return convertChar(c1) == convertChar(c2); }); } - static bool less(const std::string_view & s1, const std::string_view & s2) + static bool less(std::string_view s1, std::string_view s2) { return std::lexicographical_compare(s1.begin(), s1.end(), s2.begin(), s2.end(), [](char c1, char c2) { return convertChar(c1) < convertChar(c2); }); } - static bool startsWith(const std::string_view & s1, const std::string_view & s2) + static bool startsWith(std::string_view s1, std::string_view s2) { return (s1.length() >= s2.length()) && equals(s1.substr(0, s2.length()), s2); } @@ -195,7 +195,7 @@ namespace { protected: ProtobufSerializerSingleValue( - const std::string_view & column_name_, + std::string_view column_name_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) : column_name(column_name_) @@ -264,7 +264,7 @@ namespace return reader->readFixed(); } - void writeStr(const std::string_view & str) + void writeStr(std::string_view str) { if (!str.empty() || !skip_zero_or_empty) writer->writeString(field_tag, str); @@ -274,7 +274,7 @@ namespace void readStrAndAppend(PaddedPODArray & str) { reader->readStringAndAppend(str); } template - DestType parseFromStr(const std::string_view & str) const + DestType parseFromStr(std::string_view str) const { try { @@ -307,7 +307,7 @@ namespace return result; } - [[noreturn]] void incompatibleColumnType(const std::string_view & column_type) const + [[noreturn]] void incompatibleColumnType(std::string_view column_type) const { throw Exception( ErrorCodes::DATA_TYPE_INCOMPATIBLE_WITH_PROTOBUF_FIELD, @@ -318,7 +318,7 @@ namespace field_descriptor.type_name()); } - [[noreturn]] void cannotConvertValue(const std::string_view & src_value, const std::string_view & src_type_name, const std::string_view & dest_type_name) const + [[noreturn]] void cannotConvertValue(std::string_view src_value, std::string_view src_type_name, std::string_view dest_type_name) const { throw Exception( "Could not convert value '" + String{src_value} + "' from type " + String{src_type_name} + " to type " @@ -351,7 +351,7 @@ namespace public: using ColumnType = ColumnVector; - ProtobufSerializerNumber(const std::string_view & column_name_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) + ProtobufSerializerNumber(std::string_view column_name_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) : ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_) { setFunctions(); @@ -590,7 +590,7 @@ namespace using ColumnType = std::conditional_t; ProtobufSerializerString( - const std::string_view & column_name_, + std::string_view column_name_, const std::shared_ptr & fixed_string_data_type_, const google::protobuf::FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) @@ -604,7 +604,7 @@ namespace } ProtobufSerializerString( - const std::string_view & column_name_, + std::string_view column_name_, const google::protobuf::FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) : ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_) @@ -727,7 +727,7 @@ namespace { case FieldTypeId::TYPE_INT32: { - write_function = [this](const std::string_view & str) { writeInt(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeInt(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readInt(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); }; break; @@ -735,7 +735,7 @@ namespace case FieldTypeId::TYPE_SINT32: { - write_function = [this](const std::string_view & str) { writeSInt(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeSInt(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readSInt(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); }; break; @@ -743,7 +743,7 @@ namespace case FieldTypeId::TYPE_UINT32: { - write_function = [this](const std::string_view & str) { writeUInt(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeUInt(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readUInt(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_uint32()); }; break; @@ -751,7 +751,7 @@ namespace case FieldTypeId::TYPE_INT64: { - write_function = [this](const std::string_view & str) { writeInt(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeInt(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readInt(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); }; break; @@ -759,7 +759,7 @@ namespace case FieldTypeId::TYPE_SINT64: { - write_function = [this](const std::string_view & str) { writeSInt(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeSInt(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readSInt(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); }; break; @@ -767,7 +767,7 @@ namespace case FieldTypeId::TYPE_UINT64: { - write_function = [this](const std::string_view & str) { writeUInt(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeUInt(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readUInt(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_uint64()); }; break; @@ -775,7 +775,7 @@ namespace case FieldTypeId::TYPE_FIXED32: { - write_function = [this](const std::string_view & str) { writeFixed(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeFixed(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readFixed(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_uint32()); }; break; @@ -783,7 +783,7 @@ namespace case FieldTypeId::TYPE_SFIXED32: { - write_function = [this](const std::string_view & str) { writeFixed(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeFixed(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readFixed(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_int32()); }; break; @@ -791,7 +791,7 @@ namespace case FieldTypeId::TYPE_FIXED64: { - write_function = [this](const std::string_view & str) { writeFixed(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeFixed(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readFixed(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_uint64()); }; break; @@ -799,7 +799,7 @@ namespace case FieldTypeId::TYPE_SFIXED64: { - write_function = [this](const std::string_view & str) { writeFixed(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeFixed(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readFixed(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_int64()); }; break; @@ -807,7 +807,7 @@ namespace case FieldTypeId::TYPE_FLOAT: { - write_function = [this](const std::string_view & str) { writeFixed(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeFixed(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readFixed(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_float()); }; break; @@ -815,7 +815,7 @@ namespace case FieldTypeId::TYPE_DOUBLE: { - write_function = [this](const std::string_view & str) { writeFixed(parseFromStr(str)); }; + write_function = [this](std::string_view str) { writeFixed(parseFromStr(str)); }; read_function = [this](PaddedPODArray & str) { toStringAppend(readFixed(), str); }; default_function = [this]() -> String { return toString(field_descriptor.default_value_double()); }; break; @@ -823,7 +823,7 @@ namespace case FieldTypeId::TYPE_BOOL: { - write_function = [this](const std::string_view & str) + write_function = [this](std::string_view str) { if (str == "true") writeUInt(1); @@ -855,7 +855,7 @@ namespace case FieldTypeId::TYPE_STRING: case FieldTypeId::TYPE_BYTES: { - write_function = [this](const std::string_view & str) { writeStr(str); }; + write_function = [this](std::string_view str) { writeStr(str); }; read_function = [this](PaddedPODArray & str) { readStrAndAppend(str); }; default_function = [this]() -> String { return field_descriptor.default_value_string(); }; break; @@ -863,7 +863,7 @@ namespace case FieldTypeId::TYPE_ENUM: { - write_function = [this](const std::string_view & str) { writeInt(stringToProtobufEnumValue(str)); }; + write_function = [this](std::string_view str) { writeInt(stringToProtobufEnumValue(str)); }; read_function = [this](PaddedPODArray & str) { protobufEnumValueToStringAppend(readInt(), str); }; default_function = [this]() -> String { return field_descriptor.default_value_enum()->name(); }; break; @@ -908,7 +908,7 @@ namespace } } - int stringToProtobufEnumValue(const std::string_view & str) const + int stringToProtobufEnumValue(std::string_view str) const { auto it = string_to_protobuf_enum_value_map.find(str); if (it == string_to_protobuf_enum_value_map.end()) @@ -932,7 +932,7 @@ namespace const std::shared_ptr fixed_string_data_type; const size_t n = 0; - std::function write_function; + std::function write_function; std::function &)> read_function; std::function default_function; std::unordered_map string_to_protobuf_enum_value_map; @@ -953,7 +953,7 @@ namespace using BaseClass = ProtobufSerializerNumber; ProtobufSerializerEnum( - const std::string_view & column_name_, + std::string_view column_name_, const std::shared_ptr & enum_data_type_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) @@ -1067,7 +1067,7 @@ namespace protobuf_enum_value_to_enum_data_type_value_map.emplace(protobuf_enum_value, enum_data_type_value); }; - auto iless = [](const std::string_view & s1, const std::string_view & s2) { return ColumnNameWithProtobufFieldNameComparator::less(s1, s2); }; + auto iless = [](std::string_view s1, std::string_view s2) { return ColumnNameWithProtobufFieldNameComparator::less(s1, s2); }; boost::container::flat_map string_to_protobuf_enum_value_map; typename decltype(string_to_protobuf_enum_value_map)::sequence_type string_to_protobuf_enum_value_seq; for (int i : collections::range(enum_descriptor.value_count())) @@ -1133,9 +1133,9 @@ namespace Int64 readInt() { return ProtobufSerializerSingleValue::readInt(); } void writeInt(Int64 value) { ProtobufSerializerSingleValue::writeInt(value); } - void writeStr(const std::string_view & str) { ProtobufSerializerSingleValue::writeStr(str); } + void writeStr(std::string_view str) { ProtobufSerializerSingleValue::writeStr(str); } void readStr(String & str) { ProtobufSerializerSingleValue::readStr(str); } - [[noreturn]] void cannotConvertValue(const std::string_view & src_value, const std::string_view & src_type_name, const std::string_view & dest_type_name) const { ProtobufSerializerSingleValue::cannotConvertValue(src_value, src_type_name, dest_type_name); } + [[noreturn]] void cannotConvertValue(std::string_view src_value, std::string_view src_type_name, std::string_view dest_type_name) const { ProtobufSerializerSingleValue::cannotConvertValue(src_value, src_type_name, dest_type_name); } const std::shared_ptr enum_data_type; std::unordered_map enum_data_type_value_to_protobuf_enum_value_map; @@ -1152,7 +1152,7 @@ namespace using ColumnType = ColumnDecimal; ProtobufSerializerDecimal( - const std::string_view & column_name_, + std::string_view column_name_, const DataTypeDecimalBase & decimal_data_type_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) @@ -1412,7 +1412,7 @@ namespace { public: ProtobufSerializerDate( - const std::string_view & column_name_, + std::string_view column_name_, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) : ProtobufSerializerNumber(column_name_, field_descriptor_, reader_or_writer_) @@ -1490,7 +1490,7 @@ namespace { public: ProtobufSerializerDateTime( - const std::string_view & column_name_, + std::string_view column_name_, const DataTypeDateTime & type, const FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) @@ -1574,7 +1574,7 @@ namespace { public: ProtobufSerializerUUID( - const std::string_view & column_name_, + std::string_view column_name_, const google::protobuf::FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) : ProtobufSerializerSingleValue(column_name_, field_descriptor_, reader_or_writer_) @@ -1654,7 +1654,7 @@ namespace { public: ProtobufSerializerAggregateFunction( - const std::string_view & column_name_, + std::string_view column_name_, const std::shared_ptr & aggregate_function_data_type_, const google::protobuf::FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) @@ -2061,7 +2061,7 @@ namespace { public: ProtobufSerializerTupleAsArray( - const std::string_view & column_name_, + std::string_view column_name_, const std::shared_ptr & tuple_data_type_, const FieldDescriptor & field_descriptor_, std::vector> element_serializers_) @@ -2833,7 +2833,7 @@ namespace return field_names; } - static bool columnNameEqualsToFieldName(const std::string_view & column_name, const FieldDescriptor & field_descriptor) + static bool columnNameEqualsToFieldName(std::string_view column_name, const FieldDescriptor & field_descriptor) { std::string_view suffix; return columnNameStartsWithFieldName(column_name, field_descriptor, suffix) && suffix.empty(); @@ -2844,7 +2844,7 @@ namespace /// which doesn't match to the field's name. /// The function requires that rest part of the column's name to be started with a dot '.' or underline '_', /// but doesn't include those '.' or '_' characters into `suffix`. - static bool columnNameStartsWithFieldName(const std::string_view & column_name, const FieldDescriptor & field_descriptor, std::string_view & suffix) + static bool columnNameStartsWithFieldName(std::string_view column_name, const FieldDescriptor & field_descriptor, std::string_view & suffix) { size_t matching_length = 0; const MessageDescriptor & containing_type = *field_descriptor.containing_type(); @@ -2887,7 +2887,7 @@ namespace /// for that case suffixes are also returned. /// This is only the first filter, buildMessageSerializerImpl() does other checks after calling this function. static bool findFieldsByColumnName( - const std::string_view & column_name, + std::string_view column_name, const MessageDescriptor & message_descriptor, std::vector> & out_field_descriptors_with_suffixes, bool google_wrappers_special_treatment) @@ -3030,7 +3030,7 @@ namespace used_column_indices_sorted.reserve(num_columns); size_t sequential_column_index = 0; - auto add_field_serializer = [&](const std::string_view & column_name_, + auto add_field_serializer = [&](std::string_view column_name_, std::vector && column_indices_, const FieldDescriptor & field_descriptor_, std::unique_ptr field_serializer_) @@ -3243,7 +3243,7 @@ namespace /// Builds a serializer for one-to-one match: /// one column is serialized as one field in the protobuf message. std::unique_ptr buildFieldSerializer( - const std::string_view & column_name, + std::string_view column_name, const DataTypePtr & data_type, const FieldDescriptor & field_descriptor, bool allow_repeat, @@ -3395,7 +3395,7 @@ namespace } } - [[noreturn]] static void throwFieldNotRepeated(const FieldDescriptor & field_descriptor, const std::string_view & column_name) + [[noreturn]] static void throwFieldNotRepeated(const FieldDescriptor & field_descriptor, std::string_view column_name) { if (!field_descriptor.is_repeated()) throw Exception( diff --git a/src/Formats/ProtobufWriter.cpp b/src/Formats/ProtobufWriter.cpp index ece4f78b1c8..da680fae601 100644 --- a/src/Formats/ProtobufWriter.cpp +++ b/src/Formats/ProtobufWriter.cpp @@ -196,7 +196,7 @@ template void ProtobufWriter::writeFixed(int field_number, UInt64 value) template void ProtobufWriter::writeFixed(int field_number, Float32 value); template void ProtobufWriter::writeFixed(int field_number, Float64 value); -void ProtobufWriter::writeString(int field_number, const std::string_view & str) +void ProtobufWriter::writeString(int field_number, std::string_view str) { size_t length = str.length(); size_t old_size = buffer.size(); diff --git a/src/Formats/ProtobufWriter.h b/src/Formats/ProtobufWriter.h index 1dcc8f4ef7c..3ede956e910 100644 --- a/src/Formats/ProtobufWriter.h +++ b/src/Formats/ProtobufWriter.h @@ -30,7 +30,7 @@ public: void writeSInt(int field_number, Int64 value); template void writeFixed(int field_number, T value); - void writeString(int field_number, const std::string_view & str); + void writeString(int field_number, std::string_view str); void startRepeatedPack(); void endRepeatedPack(int field_number, bool skip_if_empty); diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index fa573ac829a..2f10a3c773b 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -304,7 +304,7 @@ private: /// Performs moves of types MoveType::Key and MoveType::ConstKey. template - static bool moveToElementByKey(typename JSONParser::Element & element, const std::string_view & key) + static bool moveToElementByKey(typename JSONParser::Element & element, std::string_view key) { if (!element.isObject()) return false; @@ -504,7 +504,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view &) + static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view) { ColumnVector & col_vec = assert_cast &>(dest); col_vec.insertValue(1); @@ -532,7 +532,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName &) { return 0; } - static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view &) + static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view) { /// This function is called only if JSON is valid. /// If JSON isn't valid then `FunctionJSON::Executor::run()` adds default value (=zero) to `dest` without calling this function. @@ -556,7 +556,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { size_t size; if (element.isArray()) @@ -586,7 +586,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view & last_key) + static bool insertResultToColumn(IColumn & dest, const Element &, std::string_view last_key) { if (last_key.empty()) return false; @@ -620,7 +620,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { UInt8 type; if (element.isInt64()) @@ -662,7 +662,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { NumberType value; @@ -737,7 +737,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { if (!element.isBool()) return false; @@ -764,7 +764,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { if (element.isNull()) return false; @@ -1164,7 +1164,7 @@ public: extract_tree = JSONExtractTree::build(function_name, result_type); } - bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { return extract_tree->insertResultToColumn(dest, element); } @@ -1207,7 +1207,7 @@ public: extract_tree = JSONExtractTree::build(function_name, value_type); } - bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { if (!element.isObject()) return false; @@ -1251,7 +1251,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { ColumnString & col_str = assert_cast(dest); auto & chars = col_str.getChars(); @@ -1355,7 +1355,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { if (!element.isArray()) return false; @@ -1387,7 +1387,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { if (!element.isObject()) return false; @@ -1423,7 +1423,7 @@ public: static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view) { if (!element.isObject()) return false; diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 866d3ae3866..964dc92ef3c 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -562,15 +562,23 @@ public: { const ColumnMap * col_map_left = typeid_cast(arguments[0].column.get()); const auto * col_const_map_left = checkAndGetColumnConst(arguments[0].column.get()); + bool col_const_map_left_flag = false; if (col_const_map_left) + { + col_const_map_left_flag = true; col_map_left = typeid_cast(&col_const_map_left->getDataColumn()); + } if (!col_map_left) return nullptr; const ColumnMap * col_map_right = typeid_cast(arguments[1].column.get()); const auto * col_const_map_right = checkAndGetColumnConst(arguments[1].column.get()); + bool col_const_map_right_flag = false; if (col_const_map_right) + { + col_const_map_right_flag = true; col_map_right = typeid_cast(&col_const_map_right->getDataColumn()); + } if (!col_map_right) return nullptr; @@ -592,13 +600,18 @@ public: MutableColumnPtr offsets = DataTypeNumber().createColumn(); IColumn::Offset current_offset = 0; - for (size_t idx = 0; idx < input_rows_count; ++idx) + for (size_t row_idx = 0; row_idx < input_rows_count; ++row_idx) { - for (size_t i = offsets_left[idx - 1]; i < offsets_left[idx]; ++i) + size_t left_it_begin = col_const_map_left_flag ? 0 : offsets_left[row_idx - 1]; + size_t left_it_end = col_const_map_left_flag ? offsets_left.size() : offsets_left[row_idx]; + size_t right_it_begin = col_const_map_right_flag ? 0 : offsets_right[row_idx - 1]; + size_t right_it_end = col_const_map_right_flag ? offsets_right.size() : offsets_right[row_idx]; + + for (size_t i = left_it_begin; i < left_it_end; ++i) { bool matched = false; auto key = keys_data_left.getDataAt(i); - for (size_t j = offsets_right[idx - 1]; j < offsets_right[idx]; ++j) + for (size_t j = right_it_begin; j < right_it_end; ++j) { if (keys_data_right.getDataAt(j).toString() == key.toString()) { @@ -613,12 +626,14 @@ public: ++current_offset; } } - for (size_t j = offsets_right[idx - 1]; j < offsets_right[idx]; ++j) + + for (size_t j = right_it_begin; j < right_it_end; ++j) { keys_data->insertFrom(keys_data_right, j); values_data->insertFrom(values_data_right, j); ++current_offset; } + offsets->insert(current_offset); } diff --git a/src/Functions/registerFunctionsStringRegexp.cpp b/src/Functions/registerFunctionsStringRegexp.cpp index 61853b19d11..df7e8f58396 100644 --- a/src/Functions/registerFunctionsStringRegexp.cpp +++ b/src/Functions/registerFunctionsStringRegexp.cpp @@ -9,6 +9,7 @@ void registerFunctionNotLike(FunctionFactory &); void registerFunctionNotILike(FunctionFactory &); void registerFunctionMatch(FunctionFactory &); void registerFunctionExtract(FunctionFactory &); +void registerFunctionTranslate(FunctionFactory &); void registerFunctionReplaceOne(FunctionFactory &); void registerFunctionReplaceAll(FunctionFactory &); void registerFunctionReplaceRegexpOne(FunctionFactory &); @@ -31,6 +32,7 @@ void registerFunctionsStringRegexp(FunctionFactory & factory) registerFunctionNotILike(factory); registerFunctionMatch(factory); registerFunctionExtract(factory); + registerFunctionTranslate(factory); registerFunctionReplaceOne(factory); registerFunctionReplaceAll(factory); registerFunctionReplaceRegexpOne(factory); diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp new file mode 100644 index 00000000000..8342bfe236b --- /dev/null +++ b/src/Functions/translate.cpp @@ -0,0 +1,364 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +struct TranslateImpl +{ + using Map = std::array; + + static void fillMapWithValues( + Map & map, + const std::string & map_from, + const std::string & map_to) + { + if (map_from.size() != map_to.size()) + throw Exception("Second and trird arguments must be the same length", ErrorCodes::BAD_ARGUMENTS); + + std::iota(map.begin(), map.end(), 0); + + for (size_t i = 0; i < map_from.size(); ++i) + { + if (!isASCII(map_from[i]) || !isASCII(map_to[i])) + throw Exception("Second and trird arguments must be ASCII strings", ErrorCodes::BAD_ARGUMENTS); + + map[map_from[i]] = map_to[i]; + } + } + + static void vector( + const ColumnString::Chars & data, + const ColumnString::Offsets & offsets, + const std::string & map_from, + const std::string & map_to, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets) + { + Map map; + fillMapWithValues(map, map_from, map_to); + + res_data.resize(data.size()); + res_offsets.assign(offsets); + + UInt8 * dst = res_data.data(); + + for (UInt64 i = 0; i < offsets.size(); ++i) + { + const UInt8 * src = data.data() + offsets[i - 1]; + const UInt8 * src_end = data.data() + offsets[i] - 1; + + while (src < src_end) + { + if (*src <= ascii_upper_bound) + *dst = map[*src]; + else + *dst = *src; + + ++src; + ++dst; + } + + /// Technically '\0' can be mapped into other character, + /// so we need to process '\0' delimiter separately + *dst++ = 0; + } + } + + static void vectorFixed( + const ColumnString::Chars & data, + size_t /*n*/, + const std::string & map_from, + const std::string & map_to, + ColumnString::Chars & res_data) + { + std::array map; + fillMapWithValues(map, map_from, map_to); + + res_data.resize(data.size()); + + const UInt8 * src = data.data(); + const UInt8 * src_end = data.data() + data.size(); + UInt8 * dst = res_data.data(); + + while (src < src_end) + { + if (*src <= ascii_upper_bound) + *dst = map[*src]; + else + *dst = *src; + + ++src; + ++dst; + } + } + +private: + static constexpr auto ascii_upper_bound = '\x7f'; +}; + +struct TranslateUTF8Impl +{ + using MapASCII = std::array; + using MapUTF8 = HashMap>; + + static void fillMapWithValues( + MapASCII & map_ascii, + MapUTF8 & map, + const std::string & map_from, + const std::string & map_to) + { + auto map_from_size = UTF8::countCodePoints(reinterpret_cast(map_from.data()), map_from.size()); + auto map_to_size = UTF8::countCodePoints(reinterpret_cast(map_to.data()), map_to.size()); + + if (map_from_size != map_to_size) + throw Exception("Second and trird arguments must be the same length", ErrorCodes::BAD_ARGUMENTS); + + std::iota(map_ascii.begin(), map_ascii.end(), 0); + + const UInt8 * map_from_ptr = reinterpret_cast(map_from.data()); + const UInt8 * map_from_end = map_from_ptr + map_from.size(); + const UInt8 * map_to_ptr = reinterpret_cast(map_to.data()); + const UInt8 * map_to_end = map_to_ptr + map_to.size(); + + while (map_from_ptr < map_from_end && map_to_ptr < map_to_end) + { + size_t len_from = UTF8::seqLength(*map_from_ptr); + size_t len_to = UTF8::seqLength(*map_to_ptr); + + std::optional res_from, res_to; + + if (map_from_ptr + len_from <= map_from_end) + res_from = UTF8::convertUTF8ToCodePoint(map_from_ptr, len_from); + + if (map_to_ptr + len_to <= map_to_end) + res_to = UTF8::convertUTF8ToCodePoint(map_to_ptr, len_to); + + if (!res_from) + throw Exception("Second argument must be a valid UTF-8 string", ErrorCodes::BAD_ARGUMENTS); + + if (!res_to) + throw Exception("Third argument must be a valid UTF-8 string", ErrorCodes::BAD_ARGUMENTS); + + if (*map_from_ptr <= ascii_upper_bound) + map_ascii[*map_from_ptr] = *res_to; + else + map[*res_from] = *res_to; + + map_from_ptr += len_from; + map_to_ptr += len_to; + } + } + + static void vector( + const ColumnString::Chars & data, + const ColumnString::Offsets & offsets, + const std::string & map_from, + const std::string & map_to, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets) + { + MapASCII map_ascii; + MapUTF8 map; + fillMapWithValues(map_ascii, map, map_from, map_to); + + res_data.resize(data.size()); + res_offsets.resize(offsets.size()); + + UInt8 * dst = res_data.data(); + UInt64 data_size = 0; + + for (UInt64 i = 0; i < offsets.size(); ++i) + { + const UInt8 * src = data.data() + offsets[i - 1]; + const UInt8 * src_end = data.data() + offsets[i] - 1; + + while (src < src_end) + { + /// Maximum length of UTF-8 sequence is 4 bytes + 1 zero byte + if (data_size + 5 > res_data.size()) + { + res_data.resize(data_size * 2 + 5); + dst = res_data.data() + data_size; + } + + if (*src <= ascii_upper_bound) + { + size_t dst_len = UTF8::convertCodePointToUTF8(map_ascii[*src], dst, 4); + assert(0 < dst_len && dst_len <= 4); + + src += 1; + dst += dst_len; + data_size += dst_len; + continue; + } + + size_t src_len = UTF8::seqLength(*src); + assert(0 < src_len && src_len <= 4); + + if (src + src_len <= src_end) + { + auto src_code_point = UTF8::convertUTF8ToCodePoint(src, src_len); + + if (src_code_point) + { + auto * it = map.find(*src_code_point); + if (it != map.end()) + { + size_t dst_len = UTF8::convertCodePointToUTF8(it->getMapped(), dst, 4); + assert(0 < dst_len && dst_len <= 4); + + src += src_len; + dst += dst_len; + data_size += dst_len; + continue; + } + } + } + else + { + src_len = src_end - src; + } + + memcpy(dst, src, src_len); + dst += src_len; + src += src_len; + data_size += src_len; + } + + /// Technically '\0' can be mapped into other character, + /// so we need to process '\0' delimiter separately + *dst++ = 0; + + ++data_size; + res_offsets[i] = data_size; + } + + res_data.resize(data_size); + } + + [[noreturn]] static void vectorFixed( + const ColumnString::Chars & /*data*/, + size_t /*n*/, + const std::string & /*map_from*/, + const std::string & /*map_to*/, + ColumnString::Chars & /*res_data*/) + { + throw Exception("Function translateUTF8 does not support FixedString argument", ErrorCodes::BAD_ARGUMENTS); + } + +private: + static constexpr auto ascii_upper_bound = '\x7f'; +}; + + +template +class FunctionTranslate : public IFunction +{ +public: + static constexpr auto name = Name::name; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 3; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (!isStringOrFixedString(arguments[0])) + throw Exception( + "Illegal type " + arguments[0]->getName() + " of first argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (!isStringOrFixedString(arguments[1])) + throw Exception( + "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (!isStringOrFixedString(arguments[2])) + throw Exception( + "Illegal type " + arguments[2]->getName() + " of third argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + { + const ColumnPtr column_src = arguments[0].column; + const ColumnPtr column_map_from = arguments[1].column; + const ColumnPtr column_map_to = arguments[2].column; + + if (!isColumnConst(*column_map_from) || !isColumnConst(*column_map_to)) + throw Exception("2nd and 3rd arguments of function " + getName() + " must be constants.", ErrorCodes::ILLEGAL_COLUMN); + + const IColumn * c1 = arguments[1].column.get(); + const IColumn * c2 = arguments[2].column.get(); + const ColumnConst * c1_const = typeid_cast(c1); + const ColumnConst * c2_const = typeid_cast(c2); + String map_from = c1_const->getValue(); + String map_to = c2_const->getValue(); + + if (const ColumnString * col = checkAndGetColumn(column_src.get())) + { + auto col_res = ColumnString::create(); + Impl::vector(col->getChars(), col->getOffsets(), map_from, map_to, col_res->getChars(), col_res->getOffsets()); + return col_res; + } + else if (const ColumnFixedString * col_fixed = checkAndGetColumn(column_src.get())) + { + auto col_res = ColumnFixedString::create(col_fixed->getN()); + Impl::vectorFixed(col_fixed->getChars(), col_fixed->getN(), map_from, map_to, col_res->getChars()); + return col_res; + } + else + throw Exception( + "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } +}; + + +namespace +{ + +struct NameTranslate +{ + static constexpr auto name = "translate"; +}; + +struct NameTranslateUTF8 +{ + static constexpr auto name = "translateUTF8"; +}; + +using FunctionTranslateASCII = FunctionTranslate; +using FunctionTranslateUTF8 = FunctionTranslate; + +} + +void registerFunctionTranslate(FunctionFactory & factory) +{ + factory.registerFunction(); + factory.registerFunction(); +} + +} diff --git a/src/IO/ReadBufferFromAzureBlobStorage.cpp b/src/IO/ReadBufferFromAzureBlobStorage.cpp index 2576b10f9ac..3e6581cd786 100644 --- a/src/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/IO/ReadBufferFromAzureBlobStorage.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -23,17 +24,18 @@ namespace ErrorCodes ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage( std::shared_ptr blob_container_client_, const String & path_, + const ReadSettings & read_settings_, size_t max_single_read_retries_, size_t max_single_download_retries_, - size_t tmp_buffer_size_, bool use_external_buffer_, size_t read_until_position_) - : SeekableReadBuffer(nullptr, 0) + : ReadBufferFromFileBase(read_settings_.remote_fs_buffer_size, nullptr, 0) , blob_container_client(blob_container_client_) , path(path_) , max_single_read_retries(max_single_read_retries_) , max_single_download_retries(max_single_download_retries_) - , tmp_buffer_size(tmp_buffer_size_) + , read_settings(read_settings_) + , tmp_buffer_size(read_settings.remote_fs_buffer_size) , use_external_buffer(use_external_buffer_) , read_until_position(read_until_position_) { @@ -75,6 +77,8 @@ bool ReadBufferFromAzureBlobStorage::nextImpl() try { bytes_read = data_stream->ReadToCount(reinterpret_cast(data_ptr), to_read_bytes); + if (read_settings.remote_throttler) + read_settings.remote_throttler->add(bytes_read); break; } catch (const Azure::Storage::StorageException & e) diff --git a/src/IO/ReadBufferFromAzureBlobStorage.h b/src/IO/ReadBufferFromAzureBlobStorage.h index e5a39b84d45..5396fcf9719 100644 --- a/src/IO/ReadBufferFromAzureBlobStorage.h +++ b/src/IO/ReadBufferFromAzureBlobStorage.h @@ -5,7 +5,7 @@ #if USE_AZURE_BLOB_STORAGE #include -#include +#include #include #include #include @@ -13,16 +13,16 @@ namespace DB { -class ReadBufferFromAzureBlobStorage : public SeekableReadBuffer, public WithFileName +class ReadBufferFromAzureBlobStorage : public ReadBufferFromFileBase { public: ReadBufferFromAzureBlobStorage( std::shared_ptr blob_container_client_, const String & path_, + const ReadSettings & read_settings_, size_t max_single_read_retries_, size_t max_single_download_retries_, - size_t tmp_buffer_size_, bool use_external_buffer_ = false, size_t read_until_position_ = 0); @@ -47,6 +47,7 @@ private: const String path; size_t max_single_read_retries; size_t max_single_download_retries; + ReadSettings read_settings; std::vector tmp_buffer; size_t tmp_buffer_size; bool use_external_buffer; diff --git a/src/IO/ReadBufferFromFileDecorator.cpp b/src/IO/ReadBufferFromFileDecorator.cpp index 6e803586cd6..9ac0fb4e475 100644 --- a/src/IO/ReadBufferFromFileDecorator.cpp +++ b/src/IO/ReadBufferFromFileDecorator.cpp @@ -21,9 +21,8 @@ std::string ReadBufferFromFileDecorator::getFileName() const { if (!file_name.empty()) return file_name; - if (ReadBufferFromFileBase * buffer = dynamic_cast(impl.get())) - return buffer->getFileName(); - return std::string(); + + return getFileNameFromReadBuffer(*impl); } diff --git a/src/IO/ReadBufferFromFileDescriptor.h b/src/IO/ReadBufferFromFileDescriptor.h index 96961e5a451..40b0717c8b1 100644 --- a/src/IO/ReadBufferFromFileDescriptor.h +++ b/src/IO/ReadBufferFromFileDescriptor.h @@ -61,6 +61,8 @@ public: void setProgressCallback(ContextPtr context); + size_t getFileOffsetOfBufferEnd() const override { return file_offset_of_buffer_end; } + private: /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout. bool poll(size_t timeout_microseconds); diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index cf19b6f1980..7fb432eab22 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -5,12 +5,13 @@ #include #include -#include #include #include #include +#include +#include #include #include @@ -47,7 +48,7 @@ ReadBufferFromS3::ReadBufferFromS3( size_t offset_, size_t read_until_position_, bool restricted_seek_) - : SeekableReadBuffer(nullptr, 0) + : ReadBufferFromFileBase(settings_.remote_fs_buffer_size, nullptr, 0) , client_ptr(std::move(client_ptr_)) , bucket(bucket_) , key(key_) @@ -164,6 +165,8 @@ bool ReadBufferFromS3::nextImpl() ProfileEvents::increment(ProfileEvents::ReadBufferFromS3Bytes, working_buffer.size()); offset += working_buffer.size(); + if (read_settings.remote_throttler) + read_settings.remote_throttler->add(working_buffer.size()); return true; } @@ -299,7 +302,6 @@ std::unique_ptr ReadBufferFromS3::initialize() if (outcome.IsSuccess()) { read_result = outcome.GetResultWithOwnership(); - size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size; return std::make_unique(read_result.GetBody(), buffer_size); } diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index 7e6d408ec9f..a6cb52cd273 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include @@ -26,7 +26,7 @@ namespace DB /** * Perform S3 HTTP GET request and provide response to read. */ -class ReadBufferFromS3 : public SeekableReadBuffer, public WithFileName, public WithFileSize +class ReadBufferFromS3 : public ReadBufferFromFileBase { private: std::shared_ptr client_ptr; @@ -85,8 +85,6 @@ private: /// There is different seek policy for disk seek and for non-disk seek /// (non-disk seek is applied for seekable input formats: orc, arrow, parquet). bool restricted_seek; - - std::optional file_size; }; /// Creates separate ReadBufferFromS3 for sequence of ranges of particular object diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 4cd07dddf25..57283a396d9 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -836,7 +836,7 @@ template inline T parse(const char * data, size_t size); template -inline T parseFromString(const std::string_view & str) +inline T parseFromString(std::string_view str) { return parse(str.data(), str.size()); } @@ -1338,7 +1338,7 @@ inline T parseWithSizeSuffix(const char * data, size_t size) } template -inline T parseWithSizeSuffix(const std::string_view & s) +inline T parseWithSizeSuffix(std::string_view s) { return parseWithSizeSuffix(s.data(), s.size()); } diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 78d5d6f3d65..2a2691e3c06 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -89,6 +90,9 @@ struct ReadSettings FileCachePtr remote_fs_cache; + /// Bandwidth throttler to use during reading + ThrottlerPtr remote_throttler; + size_t http_max_tries = 1; size_t http_retry_initial_backoff_ms = 100; size_t http_retry_max_backoff_ms = 1600; diff --git a/src/IO/WriteBufferFromAzureBlobStorage.cpp b/src/IO/WriteBufferFromAzureBlobStorage.cpp index 51d8bf6aba2..bc7b505cd91 100644 --- a/src/IO/WriteBufferFromAzureBlobStorage.cpp +++ b/src/IO/WriteBufferFromAzureBlobStorage.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -15,11 +16,13 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage( const String & blob_path_, size_t max_single_part_upload_size_, size_t buf_size_, + const WriteSettings & write_settings_, std::optional> attributes_) : BufferWithOwnMemory(buf_size_, nullptr, 0) , blob_container_client(blob_container_client_) , max_single_part_upload_size(max_single_part_upload_size_) , blob_path(blob_path_) + , write_settings(write_settings_) , attributes(attributes_) { } @@ -84,6 +87,9 @@ void WriteBufferFromAzureBlobStorage::nextImpl() } block_blob_client.CommitBlockList(block_ids); + + if (write_settings.remote_throttler) + write_settings.remote_throttler->add(read); } } diff --git a/src/IO/WriteBufferFromAzureBlobStorage.h b/src/IO/WriteBufferFromAzureBlobStorage.h index ef13a24abd8..0005705e68c 100644 --- a/src/IO/WriteBufferFromAzureBlobStorage.h +++ b/src/IO/WriteBufferFromAzureBlobStorage.h @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -24,6 +25,7 @@ public: const String & blob_path_, size_t max_single_part_upload_size_, size_t buf_size_, + const WriteSettings & write_settings_, std::optional> attributes_ = {}); ~WriteBufferFromAzureBlobStorage() override; @@ -36,6 +38,7 @@ private: std::shared_ptr blob_container_client; size_t max_single_part_upload_size; const String blob_path; + WriteSettings write_settings; std::optional> attributes; }; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 432304d6d5d..51f0c0d0743 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -61,6 +62,7 @@ WriteBufferFromS3::WriteBufferFromS3( std::optional> object_metadata_, size_t buffer_size_, ScheduleFunc schedule_, + const WriteSettings & write_settings_, FileCachePtr cache_) : BufferWithOwnMemory(buffer_size_, nullptr, 0) , bucket(bucket_) @@ -70,6 +72,7 @@ WriteBufferFromS3::WriteBufferFromS3( , s3_settings(s3_settings_) , object_metadata(std::move(object_metadata_)) , schedule(std::move(schedule_)) + , write_settings(write_settings_) , cache(cache_) { allocateBuffer(); @@ -121,8 +124,9 @@ void WriteBufferFromS3::nextImpl() } ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Bytes, offset()); - last_part_size += offset(); + if (write_settings.remote_throttler) + write_settings.remote_throttler->add(offset()); /// Data size exceeds singlepart upload threshold, need to use multipart upload. if (multipart_upload_id.empty() && last_part_size > s3_settings.max_single_part_upload_size) @@ -462,7 +466,6 @@ void WriteBufferFromS3::processPutRequest(PutObjectTask & task) { auto outcome = client_ptr->PutObject(task.req); bool with_pool = static_cast(schedule); - if (outcome.IsSuccess()) LOG_TRACE(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); else diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 4cdc39b80a0..e79051823c4 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -55,6 +56,7 @@ public: std::optional> object_metadata_ = std::nullopt, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, ScheduleFunc schedule_ = {}, + const WriteSettings & write_settings_ = {}, FileCachePtr cache_ = nullptr); ~WriteBufferFromS3() override; @@ -119,6 +121,8 @@ private: Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); + WriteSettings write_settings; + FileCachePtr cache; size_t current_download_offset = 0; std::optional file_segments_holder; diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 5eab75f14b1..c3bbaac097d 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -113,7 +113,7 @@ inline void writeStringBinary(const char * s, WriteBuffer & buf) writeStringBinary(StringRef{s}, buf); } -inline void writeStringBinary(const std::string_view & s, WriteBuffer & buf) +inline void writeStringBinary(std::string_view s, WriteBuffer & buf) { writeStringBinary(StringRef{s}, buf); } @@ -365,7 +365,7 @@ inline void writeJSONString(const StringRef & s, WriteBuffer & buf, const Format writeJSONString(s.data, s.data + s.size, buf, settings); } -inline void writeJSONString(const std::string_view & s, WriteBuffer & buf, const FormatSettings & settings) +inline void writeJSONString(std::string_view s, WriteBuffer & buf, const FormatSettings & settings) { writeJSONString(StringRef{s}, buf, settings); } @@ -440,7 +440,7 @@ inline void writeEscapedString(const StringRef & ref, WriteBuffer & buf) writeEscapedString(ref.data, ref.size, buf); } -inline void writeEscapedString(const std::string_view & ref, WriteBuffer & buf) +inline void writeEscapedString(std::string_view ref, WriteBuffer & buf) { writeEscapedString(ref.data(), ref.size(), buf); } @@ -478,7 +478,7 @@ inline void writeQuotedString(const StringRef & ref, WriteBuffer & buf) writeAnyQuotedString<'\''>(ref, buf); } -inline void writeQuotedString(const std::string_view & ref, WriteBuffer & buf) +inline void writeQuotedString(std::string_view ref, WriteBuffer & buf) { writeAnyQuotedString<'\''>(ref.data(), ref.data() + ref.size(), buf); } @@ -493,7 +493,7 @@ inline void writeDoubleQuotedString(const StringRef & s, WriteBuffer & buf) writeAnyQuotedString<'"'>(s, buf); } -inline void writeDoubleQuotedString(const std::string_view & s, WriteBuffer & buf) +inline void writeDoubleQuotedString(std::string_view s, WriteBuffer & buf) { writeAnyQuotedString<'"'>(s.data(), s.data() + s.size(), buf); } @@ -891,7 +891,7 @@ inline void writeBinary(const T & x, WriteBuffer & buf) { writePODBinary(x, buf) inline void writeBinary(const String & x, WriteBuffer & buf) { writeStringBinary(x, buf); } inline void writeBinary(const StringRef & x, WriteBuffer & buf) { writeStringBinary(x, buf); } -inline void writeBinary(const std::string_view & x, WriteBuffer & buf) { writeStringBinary(x, buf); } +inline void writeBinary(std::string_view x, WriteBuffer & buf) { writeStringBinary(x, buf); } inline void writeBinary(const Decimal32 & x, WriteBuffer & buf) { writePODBinary(x, buf); } inline void writeBinary(const Decimal64 & x, WriteBuffer & buf) { writePODBinary(x, buf); } inline void writeBinary(const Decimal128 & x, WriteBuffer & buf) { writePODBinary(x, buf); } @@ -1015,7 +1015,7 @@ inline void writeQuoted(const T & x, WriteBuffer & buf) { writeText(x, buf); } inline void writeQuoted(const String & x, WriteBuffer & buf) { writeQuotedString(x, buf); } -inline void writeQuoted(const std::string_view & x, WriteBuffer & buf) { writeQuotedString(x, buf); } +inline void writeQuoted(std::string_view x, WriteBuffer & buf) { writeQuotedString(x, buf); } inline void writeQuoted(const StringRef & x, WriteBuffer & buf) { writeQuotedString(x, buf); } @@ -1048,7 +1048,7 @@ inline void writeDoubleQuoted(const T & x, WriteBuffer & buf) { writeText(x, buf inline void writeDoubleQuoted(const String & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); } -inline void writeDoubleQuoted(const std::string_view & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); } +inline void writeDoubleQuoted(std::string_view x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); } inline void writeDoubleQuoted(const StringRef & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); } diff --git a/src/IO/WriteSettings.h b/src/IO/WriteSettings.h index 3464bb31664..7530b27794a 100644 --- a/src/IO/WriteSettings.h +++ b/src/IO/WriteSettings.h @@ -1,5 +1,7 @@ #pragma once +#include + namespace DB { @@ -7,6 +9,9 @@ namespace DB struct WriteSettings { bool enable_filesystem_cache_on_write_operations = false; + + /// Bandwidth throttler to use during writing + ThrottlerPtr remote_throttler; }; } diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index e25a6260787..b62690b7a3a 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -963,40 +963,24 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & // Don't need to do anything more for window functions here -- the // resulting column is added in ExpressionAnalyzer, similar to the // aggregate functions. - if (data.window_dependancy_state == WindowDependancyState::MAY_DEPEND) - data.window_function_in_subtree = true; return; } else if (node.compute_after_window_functions) { - // In this case we have window function call in subtree - // Add this function to actions index only if Data::build_expression_with_window_functions is set. - data.window_dependancy_state = WindowDependancyState::MAY_DEPEND; - for (const auto & arg : node.arguments->children) - { - data.window_function_in_subtree = false; - visit(arg, data); - // There is no point to check value of window_function_in_subtree here, - // because after window functions are computed, this variable is always false. - } - data.window_dependancy_state = WindowDependancyState::NONE; if (!data.build_expression_with_window_functions) - return; - } - else if (data.window_dependancy_state == WindowDependancyState::MAY_DEPEND) - { - // This function may depend on evaluation of window function. - // We need to check it and add it to the index only if Data::build_expression_with_window_functions is set. - bool subtree_contains_window_call = false; - for (const auto & arg : node.arguments->children) { - data.window_function_in_subtree = false; - visit(arg, data); - subtree_contains_window_call = subtree_contains_window_call || data.window_function_in_subtree; - } - data.window_function_in_subtree = subtree_contains_window_call; - if (subtree_contains_window_call && !data.build_expression_with_window_functions) + for (const auto & arg : node.arguments->children) + { + if (auto const * function = arg->as(); + function && function->name == "lambda") + { + // Lambda function is a special case. It shouldn't be visited here. + continue; + } + visit(arg, data); + } return; + } } // An aggregate function can also be calculated as a window function, but we diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index afdf2948d47..5b5a3d31da2 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -121,12 +121,6 @@ class ActionsMatcher public: using Visitor = ConstInDepthNodeVisitor; - enum class WindowDependancyState - { - NONE, - MAY_DEPEND, - }; - struct Data : public WithContext { SizeLimits set_size_limit; @@ -150,9 +144,6 @@ public: */ int next_unique_suffix; - WindowDependancyState window_dependancy_state = WindowDependancyState::NONE; - bool window_function_in_subtree = false; - Data( ContextPtr context_, SizeLimits set_size_limit_, diff --git a/src/Interpreters/ConcurrentHashJoin.cpp b/src/Interpreters/ConcurrentHashJoin.cpp index f6ba9f95bbc..5d6318a8df1 100644 --- a/src/Interpreters/ConcurrentHashJoin.cpp +++ b/src/Interpreters/ConcurrentHashJoin.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -26,19 +27,20 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int SET_SIZE_LIMIT_EXCEEDED; - extern const int BAD_ARGUMENTS; +} + +static UInt32 toPowerOfTwo(UInt32 x) +{ + if (x <= 1) + return 1; + return static_cast(1) << (32 - __builtin_clz(x - 1)); } ConcurrentHashJoin::ConcurrentHashJoin(ContextPtr context_, std::shared_ptr table_join_, size_t slots_, const Block & right_sample_block, bool any_take_last_row_) : context(context_) , table_join(table_join_) - , slots(slots_) + , slots(toPowerOfTwo(std::min(slots_, 256))) { - if (slots < 1 || 255 < slots) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Number of slots should be [1, 255], got {}", slots); - } - for (size_t i = 0; i < slots; ++i) { auto inner_hash_join = std::make_shared(); @@ -94,6 +96,7 @@ bool ConcurrentHashJoin::addJoinedBlock(const Block & right_block, bool check_li void ConcurrentHashJoin::joinBlock(Block & block, std::shared_ptr & /*not_processed*/) { Blocks dispatched_blocks = dispatchBlock(table_join->getOnlyClause().key_names_left, block); + block = {}; for (size_t i = 0; i < dispatched_blocks.size(); ++i) { std::shared_ptr none_extra_block; @@ -171,36 +174,45 @@ std::shared_ptr ConcurrentHashJoin::getNonJoinedBlocks( throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid join type. join kind: {}, strictness: {}", table_join->kind(), table_join->strictness()); } -static IColumn::Selector hashToSelector(const WeakHash32 & hash, size_t num_shards) +static ALWAYS_INLINE IColumn::Selector hashToSelector(const WeakHash32 & hash, size_t num_shards) { + assert(num_shards > 0 && (num_shards & (num_shards - 1)) == 0); const auto & data = hash.getData(); size_t num_rows = data.size(); IColumn::Selector selector(num_rows); for (size_t i = 0; i < num_rows; ++i) - selector[i] = data[i] % num_shards; + /// Apply intHash64 to mix bits in data. + /// HashTable internally uses WeakHash32, and we need to get different lower bits not to cause collisions. + selector[i] = intHash64(data[i]) & (num_shards - 1); return selector; } +IColumn::Selector ConcurrentHashJoin::selectDispatchBlock(const Strings & key_columns_names, const Block & from_block) +{ + size_t num_rows = from_block.rows(); + size_t num_shards = hash_joins.size(); + + WeakHash32 hash(num_rows); + for (const auto & key_name : key_columns_names) + { + const auto & key_col = from_block.getByName(key_name).column->convertToFullColumnIfConst(); + const auto & key_col_no_lc = recursiveRemoveLowCardinality(recursiveRemoveSparse(key_col)); + key_col_no_lc->updateWeakHash32(hash); + } + return hashToSelector(hash, num_shards); +} + Blocks ConcurrentHashJoin::dispatchBlock(const Strings & key_columns_names, const Block & from_block) { size_t num_shards = hash_joins.size(); - size_t num_rows = from_block.rows(); size_t num_cols = from_block.columns(); - WeakHash32 hash(num_rows); - for (const auto & key_name : key_columns_names) - { - const auto & key_col = from_block.getByName(key_name).column; - key_col->updateWeakHash32(hash); - } - auto selector = hashToSelector(hash, num_shards); + IColumn::Selector selector = selectDispatchBlock(key_columns_names, from_block); - Blocks result; + Blocks result(num_shards); for (size_t i = 0; i < num_shards; ++i) - { - result.emplace_back(from_block.cloneEmpty()); - } + result[i] = from_block.cloneEmpty(); for (size_t i = 0; i < num_cols; ++i) { diff --git a/src/Interpreters/ConcurrentHashJoin.h b/src/Interpreters/ConcurrentHashJoin.h index fb226c39a0c..705e6ba81b7 100644 --- a/src/Interpreters/ConcurrentHashJoin.h +++ b/src/Interpreters/ConcurrentHashJoin.h @@ -62,13 +62,10 @@ private: size_t slots; std::vector> hash_joins; - std::mutex finished_add_joined_blocks_tasks_mutex; - std::condition_variable finished_add_joined_blocks_tasks_cond; - std::atomic finished_add_joined_blocks_tasks = 0; - - mutable std::mutex totals_mutex; + std::mutex totals_mutex; Block totals; + IColumn::Selector selectDispatchBlock(const Strings & key_columns_names, const Block & from_block); Blocks dispatchBlock(const Strings & key_columns_names, const Block & from_block); }; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index b83b6420548..cbbcc58df2e 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -228,8 +228,10 @@ struct ContextSharedPart mutable std::unique_ptr distributed_schedule_pool; /// A thread pool that can run different jobs in background (used for distributed sends) mutable std::unique_ptr message_broker_schedule_pool; /// A thread pool that can run different jobs in background (used for message brokers, like RabbitMQ and Kafka) - mutable ThrottlerPtr replicated_fetches_throttler; /// A server-wide throttler for replicated fetches - mutable ThrottlerPtr replicated_sends_throttler; /// A server-wide throttler for replicated sends + mutable ThrottlerPtr replicated_fetches_throttler; /// A server-wide throttler for replicated fetches + mutable ThrottlerPtr replicated_sends_throttler; /// A server-wide throttler for replicated sends + mutable ThrottlerPtr remote_read_throttler; /// A server-wide throttler for remote IO reads + mutable ThrottlerPtr remote_write_throttler; /// A server-wide throttler for remote IO writes MultiVersion macros; /// Substitutions extracted from config. std::unique_ptr ddl_worker; /// Process ddl commands from zk. @@ -854,13 +856,13 @@ void Context::checkAccessImpl(const Args &... args) const } void Context::checkAccess(const AccessFlags & flags) const { return checkAccessImpl(flags); } -void Context::checkAccess(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl(flags, database); } -void Context::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl(flags, database, table); } -void Context::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl(flags, database, table, column); } -void Context::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return checkAccessImpl(flags, database, table, columns); } -void Context::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl(flags, database, table, columns); } +void Context::checkAccess(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl(flags, database); } +void Context::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl(flags, database, table); } +void Context::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl(flags, database, table, column); } +void Context::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const { return checkAccessImpl(flags, database, table, columns); } +void Context::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl(flags, database, table, columns); } void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id) const { checkAccessImpl(flags, table_id.getDatabaseName(), table_id.getTableName()); } -void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id, const std::string_view & column) const { checkAccessImpl(flags, table_id.getDatabaseName(), table_id.getTableName(), column); } +void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id, std::string_view column) const { checkAccessImpl(flags, table_id.getDatabaseName(), table_id.getTableName(), column); } void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id, const std::vector & columns) const { checkAccessImpl(flags, table_id.getDatabaseName(), table_id.getTableName(), columns); } void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id, const Strings & columns) const { checkAccessImpl(flags, table_id.getDatabaseName(), table_id.getTableName(), columns); } void Context::checkAccess(const AccessRightsElement & element) const { return checkAccessImpl(element); } @@ -1930,6 +1932,26 @@ ThrottlerPtr Context::getReplicatedSendsThrottler() const return shared->replicated_sends_throttler; } +ThrottlerPtr Context::getRemoteReadThrottler() const +{ + auto lock = getLock(); + if (!shared->remote_read_throttler) + shared->remote_read_throttler = std::make_shared( + settings.max_remote_read_network_bandwidth_for_server); + + return shared->remote_read_throttler; +} + +ThrottlerPtr Context::getRemoteWriteThrottler() const +{ + auto lock = getLock(); + if (!shared->remote_write_throttler) + shared->remote_write_throttler = std::make_shared( + settings.max_remote_write_network_bandwidth_for_server); + + return shared->remote_write_throttler; +} + bool Context::hasDistributedDDL() const { return getConfigRef().has("distributed_ddl"); @@ -3436,6 +3458,8 @@ ReadSettings Context::getReadSettings() const res.mmap_threshold = settings.min_bytes_to_use_mmap_io; res.priority = settings.read_priority; + res.remote_throttler = getRemoteReadThrottler(); + res.http_max_tries = settings.http_max_tries; res.http_retry_initial_backoff_ms = settings.http_retry_initial_backoff_ms; res.http_retry_max_backoff_ms = settings.http_retry_max_backoff_ms; @@ -3452,6 +3476,8 @@ WriteSettings Context::getWriteSettings() const res.enable_filesystem_cache_on_write_operations = settings.enable_filesystem_cache_on_write_operations; + res.remote_throttler = getRemoteWriteThrottler(); + return res; } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 37c6b4c9caa..ca0c218a4c0 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -474,13 +474,13 @@ public: /// Checks access rights. /// Empty database means the current database. void checkAccess(const AccessFlags & flags) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + void checkAccess(const AccessFlags & flags, std::string_view database) const; + void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const; + void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const; + void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector & columns) const; + void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const; void checkAccess(const AccessFlags & flags, const StorageID & table_id) const; - void checkAccess(const AccessFlags & flags, const StorageID & table_id, const std::string_view & column) const; + void checkAccess(const AccessFlags & flags, const StorageID & table_id, std::string_view column) const; void checkAccess(const AccessFlags & flags, const StorageID & table_id, const std::vector & columns) const; void checkAccess(const AccessFlags & flags, const StorageID & table_id, const Strings & columns) const; void checkAccess(const AccessRightsElement & element) const; @@ -820,6 +820,8 @@ public: ThrottlerPtr getReplicatedFetchesThrottler() const; ThrottlerPtr getReplicatedSendsThrottler() const; + ThrottlerPtr getRemoteReadThrottler() const; + ThrottlerPtr getRemoteWriteThrottler() const; /// Has distributed_ddl configuration or not. bool hasDistributedDDL() const; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index bae2aed2cd5..587d58d9ad3 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -234,7 +234,7 @@ void DatabaseCatalog::shutdownImpl() view_dependencies.clear(); } -bool DatabaseCatalog::isPredefinedDatabase(const std::string_view & database_name) +bool DatabaseCatalog::isPredefinedDatabase(std::string_view database_name) { return database_name == TEMPORARY_DATABASE || database_name == SYSTEM_DATABASE || database_name == INFORMATION_SCHEMA || database_name == INFORMATION_SCHEMA_UPPERCASE; diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 133cf0c5126..d82ad56eadd 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -131,7 +131,7 @@ public: static constexpr const char * INFORMATION_SCHEMA_UPPERCASE = "INFORMATION_SCHEMA"; /// Returns true if a passed name is one of the predefined databases' names. - static bool isPredefinedDatabase(const std::string_view & database_name); + static bool isPredefinedDatabase(std::string_view database_name); static DatabaseCatalog & init(ContextMutablePtr global_context_); static DatabaseCatalog & instance(); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index b79dc9c80e9..483ffad67b7 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1497,6 +1497,25 @@ void SelectQueryExpressionAnalyzer::appendExpressionsAfterWindowFunctions(Expres } } +void SelectQueryExpressionAnalyzer::appendSelectSkipWindowExpressions(ExpressionActionsChain::Step & step, ASTPtr const & node) +{ + if (auto * function = node->as()) + { + // Skip window function columns here -- they are calculated after + // other SELECT expressions by a special step. + // Also skipping lambda functions because they can't be explicitly evaluated. + if (function->is_window_function || function->name == "lambda") + return; + if (function->compute_after_window_functions) + { + for (auto & arg : function->arguments->children) + appendSelectSkipWindowExpressions(step, arg); + return; + } + } + step.addRequiredOutput(node->getColumnName()); +} + bool SelectQueryExpressionAnalyzer::appendHaving(ExpressionActionsChain & chain, bool only_types) { const auto * select_query = getAggregatingQuery(); @@ -1521,18 +1540,7 @@ void SelectQueryExpressionAnalyzer::appendSelect(ExpressionActionsChain & chain, getRootActions(select_query->select(), only_types, step.actions()); for (const auto & child : select_query->select()->children) - { - if (const auto * function = typeid_cast(child.get()); - function - && (function->is_window_function || function->compute_after_window_functions)) - { - // Skip window function columns here -- they are calculated after - // other SELECT expressions by a special step. - continue; - } - - step.addRequiredOutput(child->getColumnName()); - } + appendSelectSkipWindowExpressions(step, child); } ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order, diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 167c3dfd918..aae45482a97 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -410,6 +410,7 @@ private: void appendWindowFunctionsArguments(ExpressionActionsChain & chain, bool only_types); void appendExpressionsAfterWindowFunctions(ExpressionActionsChain & chain, bool only_types); + void appendSelectSkipWindowExpressions(ExpressionActionsChain::Step & step, ASTPtr const & node); /// After aggregation: bool appendHaving(ExpressionActionsChain & chain, bool only_types); diff --git a/src/Interpreters/FullSortingMergeJoin.h b/src/Interpreters/FullSortingMergeJoin.h index 90ec67f54b1..3ee6ce1c1fb 100644 --- a/src/Interpreters/FullSortingMergeJoin.h +++ b/src/Interpreters/FullSortingMergeJoin.h @@ -95,7 +95,7 @@ public: } /// Left and right streams have the same priority and are processed simultaneously - virtual JoinPipelineType pipelineType() const override { return JoinPipelineType::YShaped; } + JoinPipelineType pipelineType() const override { return JoinPipelineType::YShaped; } private: std::shared_ptr table_join; diff --git a/src/Interpreters/GetAggregatesVisitor.cpp b/src/Interpreters/GetAggregatesVisitor.cpp index a9d96a6d15a..dd958693d89 100644 --- a/src/Interpreters/GetAggregatesVisitor.cpp +++ b/src/Interpreters/GetAggregatesVisitor.cpp @@ -17,8 +17,7 @@ struct WindowExpressionsCollectorChildInfo // evaluation of window functions. Expression is collected only if // it's not a part of another expression. // -// Also all collected AST nodes are marked as dependent on window function. -// This information is used during ActionsDAG building process. +// Information about window function dependency is used during ActionsDAG building process. struct WindowExpressionsCollectorMatcher { using ChildInfo = WindowExpressionsCollectorChildInfo; @@ -63,11 +62,12 @@ struct WindowExpressionsCollectorMatcher result.update(subtree_result); } - // We mark functions only on the top of AST - if ((!parent || !parent->as()) && result.window_function_in_subtree) + // We mark functions if they should be computed after WindowStep + if (result.window_function_in_subtree) { - expressions_with_window_functions.push_back(func); func->compute_after_window_functions = true; + if ((!parent || !parent->as())) + expressions_with_window_functions.push_back(func); } return result; diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index a5bbcf9a373..b54c77b385f 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -411,7 +411,7 @@ HashJoin::Type HashJoin::chooseMethod(ASTTableJoin::Kind kind, const ColumnRawPt return Type::hashed; } -template +template static KeyGetter createKeyGetter(const ColumnRawPtrs & key_columns, const Sizes & key_sizes) { if constexpr (is_asof_join) diff --git a/src/Interpreters/InternalTextLogsQueue.cpp b/src/Interpreters/InternalTextLogsQueue.cpp index 2172a6f4261..8e689069cec 100644 --- a/src/Interpreters/InternalTextLogsQueue.cpp +++ b/src/Interpreters/InternalTextLogsQueue.cpp @@ -66,4 +66,19 @@ const char * InternalTextLogsQueue::getPriorityName(int priority) return (priority >= 1 && priority <= 8) ? PRIORITIES[priority] : PRIORITIES[0]; } +bool InternalTextLogsQueue::isNeeded(int priority, const String & source) const +{ + bool is_needed = priority <= max_priority; + + if (is_needed && source_regexp) + is_needed = re2::RE2::PartialMatch(source, *source_regexp); + + return is_needed; +} + +void InternalTextLogsQueue::setSourceRegexp(const String & regexp) +{ + source_regexp = std::make_unique(regexp); +} + } diff --git a/src/Interpreters/InternalTextLogsQueue.h b/src/Interpreters/InternalTextLogsQueue.h index 53710fa3bd2..9c3c9f9e707 100644 --- a/src/Interpreters/InternalTextLogsQueue.h +++ b/src/Interpreters/InternalTextLogsQueue.h @@ -2,7 +2,7 @@ #include #include #include - +#include namespace DB { @@ -15,6 +15,8 @@ public: InternalTextLogsQueue(); + bool isNeeded(int priority, const String & source) const; + static Block getSampleBlock(); static MutableColumns getSampleColumns(); @@ -23,6 +25,11 @@ public: /// Converts priority from Poco::Message::Priority to a string static const char * getPriorityName(int priority); + + void setSourceRegexp(const String & regexp); +private: + /// If not null, you should only push logs which are matched with this regexp + std::unique_ptr source_regexp; }; using InternalTextLogsQueuePtr = std::shared_ptr; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 011dc11ae97..ac31588d210 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1350,7 +1350,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

context->getTemporaryVolume(), settings.min_free_disk_space_for_temporary_data); - sorting_step->setStepDescription("Sort before JOIN"); + sorting_step->setStepDescription(fmt::format("Sort {} before JOIN", is_right ? "right" : "left")); plan.addStep(std::move(sorting_step)); }; if (expressions.join->pipelineType() == JoinPipelineType::YShaped) { const auto & join_clause = expressions.join->getTableJoin().getOnlyClause(); - add_sorting(query_plan, join_clause.key_names_left); - add_sorting(*joined_plan, join_clause.key_names_right); + add_sorting(query_plan, join_clause.key_names_left, false); + add_sorting(*joined_plan, join_clause.key_names_right, true); } QueryPlanStepPtr join_step = std::make_unique( @@ -1933,6 +1933,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc syntax_analyzer_result->optimize_trivial_count && (settings.max_parallel_replicas <= 1) && !settings.allow_experimental_query_deduplication + && !settings.empty_result_for_aggregation_by_empty_set && storage && storage->getName() != "MaterializedMySQL" && !row_policy_filter diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index f79be6a67e0..c3152f31808 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -25,17 +25,32 @@ PartLogElement::MergeReasonType PartLogElement::getMergeReasonType(MergeType mer { switch (merge_type) { - case MergeType::Regular: - return REGULAR_MERGE; - case MergeType::TTLDelete: - return TTL_DELETE_MERGE; - case MergeType::TTLRecompress: - return TTL_RECOMPRESS_MERGE; + case MergeType::Regular: + return REGULAR_MERGE; + case MergeType::TTLDelete: + return TTL_DELETE_MERGE; + case MergeType::TTLRecompress: + return TTL_RECOMPRESS_MERGE; } throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unknown MergeType {}", static_cast(merge_type)); } +PartLogElement::PartMergeAlgorithm PartLogElement::getMergeAlgorithm(MergeAlgorithm merge_algorithm_) +{ + switch (merge_algorithm_) + { + case MergeAlgorithm::Undecided: + return UNDECIDED; + case MergeAlgorithm::Horizontal: + return HORIZONTAL; + case MergeAlgorithm::Vertical: + return VERTICAL; + } + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unknown MergeAlgorithm {}", static_cast(merge_algorithm_)); +} + NamesAndTypesList PartLogElement::getNamesAndTypes() { auto event_type_datatype = std::make_shared( @@ -60,12 +75,22 @@ NamesAndTypesList PartLogElement::getNamesAndTypes() } ); + auto merge_algorithm_datatype = std::make_shared( + DataTypeEnum8::Values + { + {"Undecided", static_cast(UNDECIDED)}, + {"Horizontal", static_cast(HORIZONTAL)}, + {"Vertical", static_cast(VERTICAL)}, + } + ); + ColumnsWithTypeAndName columns_with_type_and_name; return { {"query_id", std::make_shared()}, {"event_type", std::move(event_type_datatype)}, {"merge_reason", std::move(merge_reason_datatype)}, + {"merge_algorithm", std::move(merge_algorithm_datatype)}, {"event_date", std::make_shared()}, {"event_time", std::make_shared()}, @@ -104,6 +129,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(query_id); columns[i++]->insert(event_type); columns[i++]->insert(merge_reason); + columns[i++]->insert(merge_algorithm); columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); diff --git a/src/Interpreters/PartLog.h b/src/Interpreters/PartLog.h index 16a7e37ee9d..2ce0dfd76de 100644 --- a/src/Interpreters/PartLog.h +++ b/src/Interpreters/PartLog.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -22,6 +23,14 @@ struct PartLogElement MOVE_PART = 6, }; + /// Copy of MergeAlgorithm since values are written to disk. + enum PartMergeAlgorithm + { + UNDECIDED = 0, + VERTICAL = 1, + HORIZONTAL = 2, + }; + enum MergeReasonType { /// merge_reason is relevant only for event_type = 'MERGE_PARTS', in other cases it is NOT_A_MERGE @@ -38,6 +47,7 @@ struct PartLogElement Type event_type = NEW_PART; MergeReasonType merge_reason = NOT_A_MERGE; + PartMergeAlgorithm merge_algorithm = UNDECIDED; time_t event_time = 0; Decimal64 event_time_microseconds = 0; @@ -72,6 +82,8 @@ struct PartLogElement static std::string name() { return "PartLog"; } static MergeReasonType getMergeReasonType(MergeType merge_type); + static PartMergeAlgorithm getMergeAlgorithm(MergeAlgorithm merge_algorithm_); + static NamesAndTypesList getNamesAndTypes(); static NamesAndAliases getNamesAndAliases() { return {}; } void appendToBlock(MutableColumns & columns) const; diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 1352e3777f1..17869e2084b 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -836,6 +836,8 @@ bool TableJoin::allowParallelHashJoin() const return false; if (table_join.kind != ASTTableJoin::Kind::Left && table_join.kind != ASTTableJoin::Kind::Inner) return false; + if (table_join.strictness == ASTTableJoin::Strictness::Asof) + return false; if (isSpecialStorage() || !oneDisjunct()) return false; return true; diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index b58b90b6d47..070fac7ccbd 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -31,7 +31,7 @@ namespace ErrorCodes extern const int UNSUPPORTED_JOIN_KEYS; extern const int LOGICAL_ERROR; } -bool TranslateQualifiedNamesMatcher::Data::matchColumnName(const std::string_view & name, const String & column_name, DataTypePtr column_type) +bool TranslateQualifiedNamesMatcher::Data::matchColumnName(std::string_view name, const String & column_name, DataTypePtr column_type) { if (name.size() < column_name.size()) return false; diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.h b/src/Interpreters/TranslateQualifiedNamesVisitor.h index e0c2f6b6bc0..73e45fc7ea0 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.h +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.h @@ -39,7 +39,7 @@ public: bool hasTable() const { return !tables.empty(); } bool processAsterisks() const { return hasTable() && has_columns; } bool unknownColumn(size_t table_pos, const ASTIdentifier & identifier) const; - static bool matchColumnName(const std::string_view & name, const String & column_name, DataTypePtr column_type); + static bool matchColumnName(std::string_view name, const String & column_name, DataTypePtr column_type); }; static void visit(ASTPtr & ast, Data & data); diff --git a/src/Interpreters/UserDefinedSQLObjectsLoader.cpp b/src/Interpreters/UserDefinedSQLObjectsLoader.cpp index 75b91f3a817..c6f50fc4a0a 100644 --- a/src/Interpreters/UserDefinedSQLObjectsLoader.cpp +++ b/src/Interpreters/UserDefinedSQLObjectsLoader.cpp @@ -43,7 +43,7 @@ UserDefinedSQLObjectsLoader::UserDefinedSQLObjectsLoader() : log(&Poco::Logger::get("UserDefinedSQLObjectsLoader")) {} -void UserDefinedSQLObjectsLoader::loadUserDefinedObject(ContextPtr context, UserDefinedSQLObjectType object_type, const std::string_view & name, const String & path) +void UserDefinedSQLObjectsLoader::loadUserDefinedObject(ContextPtr context, UserDefinedSQLObjectType object_type, std::string_view name, const String & path) { auto name_ref = StringRef(name.data(), name.size()); LOG_DEBUG(log, "Loading user defined object {} from file {}", backQuote(name_ref), path); diff --git a/src/Interpreters/UserDefinedSQLObjectsLoader.h b/src/Interpreters/UserDefinedSQLObjectsLoader.h index 2e747f67a8d..9dfba1181c1 100644 --- a/src/Interpreters/UserDefinedSQLObjectsLoader.h +++ b/src/Interpreters/UserDefinedSQLObjectsLoader.h @@ -29,7 +29,7 @@ public: private: - void loadUserDefinedObject(ContextPtr context, UserDefinedSQLObjectType object_type, const std::string_view & object_name, const String & file_path); + void loadUserDefinedObject(ContextPtr context, UserDefinedSQLObjectType object_type, std::string_view object_name, const String & file_path); Poco::Logger * log; bool enable_persistence = true; }; diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp index 71be8007d85..355b733b624 100644 --- a/src/Loggers/OwnSplitChannel.cpp +++ b/src/Loggers/OwnSplitChannel.cpp @@ -24,7 +24,7 @@ void OwnSplitChannel::log(const Poco::Message & msg) #ifdef WITH_TEXT_LOG auto logs_queue = CurrentThread::getInternalTextLogsQueue(); - if (channels.empty() && (logs_queue == nullptr || msg.getPriority() > logs_queue->max_priority)) + if (channels.empty() && (logs_queue == nullptr || !logs_queue->isNeeded(msg.getPriority(), msg.getSource()))) return; #endif @@ -93,7 +93,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg) auto logs_queue = CurrentThread::getInternalTextLogsQueue(); /// Log to "TCP queue" if message is not too noisy - if (logs_queue && msg.getPriority() <= logs_queue->max_priority) + if (logs_queue && logs_queue->isNeeded(msg.getPriority(), msg.getSource())) { MutableColumns columns = InternalTextLogsQueue::getSampleColumns(); diff --git a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp index 83156c6a8e1..2c25fc14e7d 100644 --- a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp @@ -75,7 +75,7 @@ namespace { for (auto filter_type : collections::range(RowPolicyFilterType::MAX)) { - const std::string_view & command = RowPolicyFilterTypeInfo::get(filter_type).command; + std::string_view command = RowPolicyFilterTypeInfo::get(filter_type).command; commands.emplace(command); } } @@ -96,7 +96,7 @@ namespace for (auto filter_type : collections::range(RowPolicyFilterType::MAX)) { - const std::string_view & command = RowPolicyFilterTypeInfo::get(filter_type).command; + std::string_view command = RowPolicyFilterTypeInfo::get(filter_type).command; if (ParserKeyword{command.data()}.ignore(pos, expected)) { res_commands.emplace(command); diff --git a/src/Processors/Executors/CompletedPipelineExecutor.cpp b/src/Processors/Executors/CompletedPipelineExecutor.cpp index 8876b3a0fb4..9e5ea3916bc 100644 --- a/src/Processors/Executors/CompletedPipelineExecutor.cpp +++ b/src/Processors/Executors/CompletedPipelineExecutor.cpp @@ -72,9 +72,11 @@ void CompletedPipelineExecutor::execute() data->executor = std::make_shared(pipeline.processors, pipeline.process_list_element); data->executor->setReadProgressCallback(pipeline.getReadProgressCallback()); - auto func = [&, thread_group = CurrentThread::getGroup()]() + /// Avoid passing this to labmda, copy ptr to data instead. + /// Destructor of unique_ptr copy raw ptr into local variable first, only then calls object destructor. + auto func = [data_ptr = data.get(), num_threads = pipeline.getNumThreads(), thread_group = CurrentThread::getGroup()]() { - threadFunction(*data, thread_group, pipeline.getNumThreads()); + threadFunction(*data_ptr, thread_group, num_threads); }; data->thread = ThreadFromGlobalPool(std::move(func)); diff --git a/src/Processors/Executors/ExecutionThreadContext.cpp b/src/Processors/Executors/ExecutionThreadContext.cpp index 5a5c1826c61..7631cb09f61 100644 --- a/src/Processors/Executors/ExecutionThreadContext.cpp +++ b/src/Processors/Executors/ExecutionThreadContext.cpp @@ -71,7 +71,13 @@ static void executeJob(ExecutingGraph::Node * node, ReadProgressCallback * read_ bool ExecutionThreadContext::executeTask() { - OpenTelemetrySpanHolder span("ExecutionThreadContext::executeTask() " + node->processor->getName()); + std::unique_ptr span; + + if (trace_processors) + { + span = std::make_unique("ExecutionThreadContext::executeTask() " + node->processor->getName()); + span->addAttribute("thread_number", thread_number); + } std::optional execution_time_watch; #ifndef NDEBUG @@ -93,17 +99,16 @@ bool ExecutionThreadContext::executeTask() if (profile_processors) { - UInt64 elapsed_microseconds = execution_time_watch->elapsedMicroseconds(); - node->processor->elapsed_us += elapsed_microseconds; - span.addAttribute("execution_time_ms", elapsed_microseconds); - } + UInt64 elapsed_microseconds = execution_time_watch->elapsedMicroseconds(); + node->processor->elapsed_us += elapsed_microseconds; + if (trace_processors) + span->addAttribute("execution_time_ms", elapsed_microseconds); + } #ifndef NDEBUG execution_time_ns += execution_time_watch->elapsed(); - span.addAttribute("execution_time_ns", execution_time_watch->elapsed()); + if (trace_processors) + span->addAttribute("execution_time_ns", execution_time_watch->elapsed()); #endif - - span.addAttribute("thread_number", thread_number); - return node->exception == nullptr; } diff --git a/src/Processors/Executors/ExecutionThreadContext.h b/src/Processors/Executors/ExecutionThreadContext.h index f0341333117..eb048f8ab09 100644 --- a/src/Processors/Executors/ExecutionThreadContext.h +++ b/src/Processors/Executors/ExecutionThreadContext.h @@ -41,6 +41,7 @@ public: const size_t thread_number; const bool profile_processors; + const bool trace_processors; void wait(std::atomic_bool & finished); void wakeUp(); @@ -61,10 +62,11 @@ public: void setException(std::exception_ptr exception_) { exception = exception_; } void rethrowExceptionIfHas(); - explicit ExecutionThreadContext(size_t thread_number_, bool profile_processors_, ReadProgressCallback * callback) + explicit ExecutionThreadContext(size_t thread_number_, bool profile_processors_, bool trace_processors_, ReadProgressCallback * callback) : read_progress_callback(callback) , thread_number(thread_number_) , profile_processors(profile_processors_) + , trace_processors(trace_processors_) {} }; diff --git a/src/Processors/Executors/ExecutorTasks.cpp b/src/Processors/Executors/ExecutorTasks.cpp index f2287e467dc..824b4e962d2 100644 --- a/src/Processors/Executors/ExecutorTasks.cpp +++ b/src/Processors/Executors/ExecutorTasks.cpp @@ -128,7 +128,7 @@ void ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThrea } } -void ExecutorTasks::init(size_t num_threads_, bool profile_processors, ReadProgressCallback * callback) +void ExecutorTasks::init(size_t num_threads_, bool profile_processors, bool trace_processors, ReadProgressCallback * callback) { num_threads = num_threads_; threads_queue.init(num_threads); @@ -139,7 +139,7 @@ void ExecutorTasks::init(size_t num_threads_, bool profile_processors, ReadProgr executor_contexts.reserve(num_threads); for (size_t i = 0; i < num_threads; ++i) - executor_contexts.emplace_back(std::make_unique(i, profile_processors, callback)); + executor_contexts.emplace_back(std::make_unique(i, profile_processors, trace_processors, callback)); } } diff --git a/src/Processors/Executors/ExecutorTasks.h b/src/Processors/Executors/ExecutorTasks.h index caff1a35d98..668470e7b11 100644 --- a/src/Processors/Executors/ExecutorTasks.h +++ b/src/Processors/Executors/ExecutorTasks.h @@ -54,7 +54,7 @@ public: void tryGetTask(ExecutionThreadContext & context); void pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context); - void init(size_t num_threads_, bool profile_processors, ReadProgressCallback * callback); + void init(size_t num_threads_, bool profile_processors, bool trace_processors, ReadProgressCallback * callback); void fill(Queue & queue); void processAsyncTasks(); diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index cccd08b2273..68225d73ff1 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -29,8 +29,10 @@ PipelineExecutor::PipelineExecutor(Processors & processors, QueryStatus * elem) : process_list_element(elem) { if (process_list_element) + { profile_processors = process_list_element->getContext()->getSettingsRef().log_processors_profiles; - + trace_processors = process_list_element->getContext()->getSettingsRef().opentelemetry_trace_processors; + } try { graph = std::make_unique(processors, profile_processors); @@ -268,7 +270,7 @@ void PipelineExecutor::initializeExecution(size_t num_threads) Queue queue; graph->initializeExecution(queue); - tasks.init(num_threads, profile_processors, read_progress_callback.get()); + tasks.init(num_threads, profile_processors, trace_processors, read_progress_callback.get()); tasks.fill(queue); } diff --git a/src/Processors/Executors/PipelineExecutor.h b/src/Processors/Executors/PipelineExecutor.h index 80ba21a8adf..c4d11ef688d 100644 --- a/src/Processors/Executors/PipelineExecutor.h +++ b/src/Processors/Executors/PipelineExecutor.h @@ -65,6 +65,8 @@ private: bool is_execution_initialized = false; /// system.processors_profile_log bool profile_processors = false; + /// system.opentelemetry_span_log + bool trace_processors = false; std::atomic_bool cancelled = false; diff --git a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp index 03a3ea99b28..3bd0fd7e3d6 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp @@ -77,7 +77,6 @@ void TabSeparatedRowOutputFormat::writeBeforeExtremes() writeChar('\n', out); } - void registerOutputFormatTabSeparated(FormatFactory & factory) { for (bool is_raw : {false, true}) diff --git a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h index 8aac94812e2..8781b7be0b1 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h +++ b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index d481c745798..909933fbed2 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -35,11 +35,22 @@ QueryPipelineBuilderPtr JoinStep::updatePipeline(QueryPipelineBuilders pipelines if (join->pipelineType() == JoinPipelineType::YShaped) return QueryPipelineBuilder::joinPipelinesYShaped( - std::move(pipelines[0]), std::move(pipelines[1]), - join, output_stream->header, - max_block_size, &processors); + std::move(pipelines[0]), std::move(pipelines[1]), join, output_stream->header, max_block_size, &processors); - return QueryPipelineBuilder::joinPipelinesRightLeft(std::move(pipelines[0]), std::move(pipelines[1]), join, max_block_size, max_streams, keep_left_read_in_order, &processors); + return QueryPipelineBuilder::joinPipelinesRightLeft( + std::move(pipelines[0]), + std::move(pipelines[1]), + join, + output_stream->header, + max_block_size, + max_streams, + keep_left_read_in_order, + &processors); +} + +bool JoinStep::allowPushDownToRight() const +{ + return join->pipelineType() == JoinPipelineType::YShaped; } void JoinStep::describePipeline(FormatSettings & settings) const @@ -47,12 +58,20 @@ void JoinStep::describePipeline(FormatSettings & settings) const IQueryPlanStep::describePipeline(processors, settings); } -void JoinStep::updateLeftStream(const DataStream & left_stream_) +void JoinStep::updateInputStream(const DataStream & new_input_stream_, size_t idx) { - input_streams = {left_stream_, input_streams.at(1)}; - output_stream = DataStream{ - .header = JoiningTransform::transformHeader(left_stream_.header, join), - }; + if (idx == 0) + { + input_streams = {new_input_stream_, input_streams.at(1)}; + output_stream = DataStream + { + .header = JoiningTransform::transformHeader(new_input_stream_.header, join), + }; + } + else + { + input_streams = {input_streams.at(0), new_input_stream_}; + } } static ITransformingStep::Traits getStorageJoinTraits() @@ -98,7 +117,7 @@ void FilledJoinStep::transformPipeline(QueryPipelineBuilder & pipeline, const Bu { bool on_totals = stream_type == QueryPipelineBuilder::StreamType::Totals; auto counter = on_totals ? nullptr : finish_counter; - return std::make_shared(header, join, max_block_size, on_totals, default_totals, counter); + return std::make_shared(header, output_stream->header, join, max_block_size, on_totals, default_totals, counter); }); } @@ -108,5 +127,4 @@ void FilledJoinStep::updateOutputStream() input_streams.front(), JoiningTransform::transformHeader(input_streams.front().header, join), getDataStreamTraits()); } - } diff --git a/src/Processors/QueryPlan/JoinStep.h b/src/Processors/QueryPlan/JoinStep.h index 17a0cc2ae63..fc7f74d4fe8 100644 --- a/src/Processors/QueryPlan/JoinStep.h +++ b/src/Processors/QueryPlan/JoinStep.h @@ -1,4 +1,5 @@ #pragma once + #include #include @@ -27,8 +28,9 @@ public: void describePipeline(FormatSettings & settings) const override; const JoinPtr & getJoin() const { return join; } + bool allowPushDownToRight() const; - void updateLeftStream(const DataStream & left_stream_); + void updateInputStream(const DataStream & new_input_stream_, size_t idx); private: JoinPtr join; diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 0c17c27e7aa..680d158ecaf 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -1,3 +1,10 @@ +#include + +#include +#include + +#include + #include #include #include @@ -11,13 +18,10 @@ #include #include #include + #include #include #include -#include -#include - -#include namespace DB::ErrorCodes { @@ -39,7 +43,8 @@ static bool filterColumnIsNotAmongAggregatesArguments(const AggregateDescription } static size_t -tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, const Names & allowed_inputs, bool can_remove_filter = true) +tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, const Names & allowed_inputs, + bool can_remove_filter = true, size_t child_idx = 0) { QueryPlan::Node * child_node = parent_node->children.front(); @@ -53,7 +58,11 @@ tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, con // std::cerr << "Filter: \n" << expression->dumpDAG() << std::endl; - const auto & all_inputs = child->getInputStreams().front().header.getColumnsWithTypeAndName(); + if (child_idx >= child->getInputStreams().size() || child_idx >= child_node->children.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Child index {} is out of range (streams: {}, children: {})", + child_idx, child->getInputStreams().size(), child_node->children.size()); + + const auto & all_inputs = child->getInputStreams()[child_idx].header.getColumnsWithTypeAndName(); auto split_filter = expression->cloneActionsForFilterPushDown(filter_column_name, removes_filter, allowed_inputs, all_inputs); if (!split_filter) @@ -75,7 +84,8 @@ tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, con /// Expression/Filter -> Aggregating -> Something auto & node = nodes.emplace_back(); node.children.emplace_back(&node); - std::swap(node.children[0], child_node->children[0]); + + std::swap(node.children[0], child_node->children[child_idx]); /// Expression/Filter -> Aggregating -> Filter -> Something /// New filter column is the first one. @@ -90,7 +100,9 @@ tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, con else { if (auto * join = typeid_cast(child.get())) - join->updateLeftStream(node.step->getOutputStream()); + { + join->updateInputStream(node.step->getOutputStream(), child_idx); + } else throw Exception( ErrorCodes::LOGICAL_ERROR, "We are trying to push down a filter through a step for which we cannot update input stream"); @@ -208,25 +220,29 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (auto * join = typeid_cast(child.get())) { - const auto & table_join = join->getJoin()->getTableJoin(); - /// Push down is for left table only. We need to update JoinStep for push down into right. - /// Only inner and left join are supported. Other types may generate default values for left table keys. - /// So, if we push down a condition like `key != 0`, not all rows may be filtered. - if (table_join.kind() == ASTTableJoin::Kind::Inner || table_join.kind() == ASTTableJoin::Kind::Left) + auto join_push_down = [&](ASTTableJoin::Kind kind) -> size_t { - const auto & left_header = join->getInputStreams().front().header; + const auto & table_join = join->getJoin()->getTableJoin(); + + /// Only inner and left(/right) join are supported. Other types may generate default values for left table keys. + /// So, if we push down a condition like `key != 0`, not all rows may be filtered. + if (table_join.kind() != ASTTableJoin::Kind::Inner && table_join.kind() != kind) + return 0; + + bool is_left = kind == ASTTableJoin::Kind::Left; + const auto & input_header = is_left ? join->getInputStreams().front().header : join->getInputStreams().back().header; const auto & res_header = join->getOutputStream().header; Names allowed_keys; - const auto & source_columns = left_header.getNames(); + const auto & source_columns = input_header.getNames(); for (const auto & name : source_columns) { /// Skip key if it is renamed. /// I don't know if it is possible. Just in case. - if (!left_header.has(name) || !res_header.has(name)) + if (!input_header.has(name) || !res_header.has(name)) continue; /// Skip if type is changed. Push down expression expect equal types. - if (!left_header.getByName(name).type->equals(*res_header.getByName(name).type)) + if (!input_header.getByName(name).type->equals(*res_header.getByName(name).type)) continue; allowed_keys.push_back(name); @@ -234,7 +250,21 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes const bool can_remove_filter = std::find(source_columns.begin(), source_columns.end(), filter->getFilterColumnName()) == source_columns.end(); - if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_keys, can_remove_filter)) + size_t updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_keys, can_remove_filter, is_left ? 0 : 1); + if (updated_steps > 0) + { + LOG_DEBUG(&Poco::Logger::get("tryPushDownFilter"), "Pushed down filter to {} side of join", kind); + } + return updated_steps; + }; + + if (size_t updated_steps = join_push_down(ASTTableJoin::Kind::Left)) + return updated_steps; + + /// For full sorting merge join we push down both to the left and right tables, because left and right streams are not independent. + if (join->allowPushDownToRight()) + { + if (size_t updated_steps = join_push_down(ASTTableJoin::Kind::Right)) return updated_steps; } } diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index 4f923810715..64343946ff4 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -23,13 +23,14 @@ Block JoiningTransform::transformHeader(Block header, const JoinPtr & join) } JoiningTransform::JoiningTransform( - Block input_header, + const Block & input_header, + const Block & output_header, JoinPtr join_, size_t max_block_size_, bool on_totals_, bool default_totals_, FinishCounterPtr finish_counter_) - : IProcessor({input_header}, {transformHeader(input_header, join_)}) + : IProcessor({input_header}, {output_header}) , join(std::move(join_)) , on_totals(on_totals_) , default_totals(default_totals_) diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h index 3383849b41d..0595d035657 100644 --- a/src/Processors/Transforms/JoiningTransform.h +++ b/src/Processors/Transforms/JoiningTransform.h @@ -39,7 +39,8 @@ public: using FinishCounterPtr = std::shared_ptr; JoiningTransform( - Block input_header, + const Block & input_header, + const Block & output_header, JoinPtr join_, size_t max_block_size_, bool on_totals_ = false, diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index 690f751209f..c7b7afab541 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -855,7 +855,7 @@ MergeJoinTransform::MergeJoinTransform( void MergeJoinTransform::onFinish() { - algorithm.logElapsed(total_stopwatch.elapsedSeconds(), true); + algorithm.logElapsed(total_stopwatch.elapsedSeconds()); } } diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 0098e470e6f..9f60eafb455 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -233,19 +233,14 @@ public: virtual void consume(Input & input, size_t source_num) override; virtual Status merge() override; - void logElapsed(double seconds, bool force) + void logElapsed(double seconds) { - /// Do not log more frequently than once per ten seconds - if (seconds - stat.last_log_seconds < 10 && !force) - return; - LOG_TRACE(log, "Finished pocessing in {} seconds" ", left: {} blocks, {} rows; right: {} blocks, {} rows" ", max blocks loaded to memory: {}", seconds, stat.num_blocks[0], stat.num_rows[0], stat.num_blocks[1], stat.num_rows[1], stat.max_blocks_loaded); - stat.last_log_seconds = seconds; } private: @@ -277,8 +272,6 @@ private: size_t num_rows[2] = {0, 0}; size_t max_blocks_loaded = 0; - - double last_log_seconds = 0; }; Statistic stat; @@ -303,12 +296,6 @@ public: protected: void onFinish() override; - void work() override - { - algorithm.logElapsed(total_stopwatch.elapsedSeconds(), true); - Base::work(); - } - Poco::Logger * log; }; diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index 5f83037e8b0..88a52defa1e 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -361,6 +361,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe std::unique_ptr left, std::unique_ptr right, JoinPtr join, + const Block & output_header, size_t max_block_size, size_t max_streams, bool keep_left_read_in_order, @@ -450,7 +451,8 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe for (size_t i = 0; i < num_streams; ++i) { - auto joining = std::make_shared(left->getHeader(), join, max_block_size, false, default_totals, finish_counter); + auto joining = std::make_shared( + left->getHeader(), output_header, join, max_block_size, false, default_totals, finish_counter); connect(**lit, joining->getInputs().front()); connect(**rit, joining->getInputs().back()); *lit = &joining->getOutputs().front(); @@ -466,7 +468,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe if (left->hasTotals()) { - auto joining = std::make_shared(left->getHeader(), join, max_block_size, true, default_totals); + auto joining = std::make_shared(left->getHeader(), output_header, join, max_block_size, true, default_totals); connect(*left->pipe.totals_port, joining->getInputs().front()); connect(**rit, joining->getInputs().back()); left->pipe.totals_port = &joining->getOutputs().front(); diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h index 18e9d718445..2d9b8028627 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.h +++ b/src/QueryPipeline/QueryPipelineBuilder.h @@ -114,6 +114,7 @@ public: std::unique_ptr left, std::unique_ptr right, JoinPtr join, + const Block & output_header, size_t max_block_size, size_t max_streams, bool keep_left_read_in_order, diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index e1c73b7ebbb..4178d0d62da 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -848,6 +848,7 @@ namespace { logs_queue = std::make_shared(); logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); + logs_queue->setSourceRegexp(settings.send_logs_source_regexp); CurrentThread::attachInternalTextLogsQueue(logs_queue, client_logs_level); CurrentThread::setFatalErrorCallback([this]{ onFatalError(); }); } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index d1d762d3c61..05565063893 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -241,6 +241,7 @@ void TCPHandler::runImpl() { state.logs_queue = std::make_shared(); state.logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); + state.logs_queue->setSourceRegexp(query_context->getSettingsRef().send_logs_source_regexp); CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level); CurrentThread::setFatalErrorCallback([this] { diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.cpp b/src/Storages/HDFS/ReadBufferFromHDFS.cpp index 208c8018c64..fab810a1e49 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.cpp +++ b/src/Storages/HDFS/ReadBufferFromHDFS.cpp @@ -2,6 +2,7 @@ #if USE_HDFS #include +#include #include #include @@ -24,13 +25,13 @@ ReadBufferFromHDFS::~ReadBufferFromHDFS() = default; struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory { - String hdfs_uri; String hdfs_file_path; hdfsFile fin; HDFSBuilderWrapper builder; HDFSFSPtr fs; + ReadSettings read_settings; off_t file_offset = 0; off_t read_until_position = 0; @@ -39,11 +40,13 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory(buf_size_) + const ReadSettings & read_settings_, + size_t read_until_position_) + : BufferWithOwnMemory(read_settings_.remote_fs_buffer_size) , hdfs_uri(hdfs_uri_) , hdfs_file_path(hdfs_file_path_) , builder(createHDFSBuilder(hdfs_uri_, config_)) + , read_settings(read_settings_) , read_until_position(read_until_position_) { fs = createHDFSFS(builder.get()); @@ -97,6 +100,8 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemoryadd(bytes_read); return true; } @@ -126,9 +131,11 @@ ReadBufferFromHDFS::ReadBufferFromHDFS( const String & hdfs_uri_, const String & hdfs_file_path_, const Poco::Util::AbstractConfiguration & config_, - size_t buf_size_, size_t read_until_position_) - : SeekableReadBuffer(nullptr, 0) - , impl(std::make_unique(hdfs_uri_, hdfs_file_path_, config_, buf_size_, read_until_position_)) + const ReadSettings & read_settings_, + size_t read_until_position_) + : ReadBufferFromFileBase(read_settings_.remote_fs_buffer_size, nullptr, 0) + , impl(std::make_unique( + hdfs_uri_, hdfs_file_path_, config_, read_settings_, read_until_position_)) { } diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.h b/src/Storages/HDFS/ReadBufferFromHDFS.h index e2929d60464..41493c31882 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.h +++ b/src/Storages/HDFS/ReadBufferFromHDFS.h @@ -11,8 +11,7 @@ #include #include #include -#include -#include +#include namespace DB @@ -20,15 +19,17 @@ namespace DB /** Accepts HDFS path to file and opens it. * Closes file by himself (thus "owns" a file descriptor). */ -class ReadBufferFromHDFS : public SeekableReadBuffer, public WithFileName, public WithFileSize +class ReadBufferFromHDFS : public ReadBufferFromFileBase { struct ReadBufferFromHDFSImpl; public: - ReadBufferFromHDFS(const String & hdfs_uri_, const String & hdfs_file_path_, - const Poco::Util::AbstractConfiguration & config_, - size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE, - size_t read_until_position_ = 0); + ReadBufferFromHDFS( + const String & hdfs_uri_, + const String & hdfs_file_path_, + const Poco::Util::AbstractConfiguration & config_, + const ReadSettings & read_settings_, + size_t read_until_position_ = 0); ~ReadBufferFromHDFS() override; diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 5e811f8e42c..57e893e9683 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -198,9 +199,9 @@ ColumnsDescription StorageHDFS::getTableStructureFromData( if (it == paths.end()) return nullptr; auto compression = chooseCompressionMethod(*it, compression_method); - auto zstd_window_log_max = ctx->getSettingsRef().zstd_window_log_max; - return wrapReadBufferWithCompressionMethod( - std::make_unique(uri_without_path, *it++, ctx->getGlobalContext()->getConfigRef()), compression, zstd_window_log_max); + auto impl = std::make_unique(uri_without_path, *it++, ctx->getGlobalContext()->getConfigRef(), ctx->getReadSettings()); + const auto zstd_window_log_max = ctx->getSettingsRef().zstd_window_log_max; + return wrapReadBufferWithCompressionMethod(std::move(impl), compression, zstd_window_log_max); }; return readSchemaFromFormat(format, std::nullopt, read_buffer_iterator, paths.size() > 1, ctx); } @@ -329,8 +330,10 @@ bool HDFSSource::initialize() const auto [path_from_uri, uri_without_path] = getPathFromUriAndUriWithoutPath(current_path); auto compression = chooseCompressionMethod(path_from_uri, storage->compression_method); + auto impl = std::make_unique( + uri_without_path, path_from_uri, getContext()->getGlobalContext()->getConfigRef(), getContext()->getReadSettings()); const auto zstd_window_log_max = getContext()->getSettingsRef().zstd_window_log_max; - read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(uri_without_path, path_from_uri, getContext()->getGlobalContext()->getConfigRef()), compression, zstd_window_log_max); + read_buf = wrapReadBufferWithCompressionMethod(std::move(impl), compression, zstd_window_log_max); auto input_format = getContext()->getInputFormat(storage->format_name, *read_buf, block_for_format, max_block_size); @@ -410,7 +413,13 @@ public: const CompressionMethod compression_method) : SinkToStorage(sample_block) { - write_buf = wrapWriteBufferWithCompressionMethod(std::make_unique(uri, context->getGlobalContext()->getConfigRef(), context->getSettingsRef().hdfs_replication), compression_method, 3); + write_buf = wrapWriteBufferWithCompressionMethod( + std::make_unique( + uri, + context->getGlobalContext()->getConfigRef(), + context->getSettingsRef().hdfs_replication, + context->getWriteSettings()), + compression_method, 3); writer = FormatFactory::instance().getOutputFormatParallelIfPossible(format, *write_buf, sample_block, context); } diff --git a/src/Storages/HDFS/WriteBufferFromHDFS.cpp b/src/Storages/HDFS/WriteBufferFromHDFS.cpp index 42ec3962beb..f8079d95f3c 100644 --- a/src/Storages/HDFS/WriteBufferFromHDFS.cpp +++ b/src/Storages/HDFS/WriteBufferFromHDFS.cpp @@ -4,9 +4,9 @@ #include #include +#include #include - namespace DB { @@ -24,15 +24,18 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl hdfsFile fout; HDFSBuilderWrapper builder; HDFSFSPtr fs; + WriteSettings write_settings; WriteBufferFromHDFSImpl( const std::string & hdfs_uri_, const Poco::Util::AbstractConfiguration & config_, int replication_, + const WriteSettings & write_settings_, int flags) : hdfs_uri(hdfs_uri_) , builder(createHDFSBuilder(hdfs_uri, config_)) , fs(createHDFSFS(builder.get())) + , write_settings(write_settings_) { const size_t begin_of_path = hdfs_uri.find('/', hdfs_uri.find("//") + 2); const String path = hdfs_uri.substr(begin_of_path); @@ -44,7 +47,6 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl throw Exception("Unable to open HDFS file: " + path + " error: " + std::string(hdfsGetLastError()), ErrorCodes::CANNOT_OPEN_FILE); } - } ~WriteBufferFromHDFSImpl() @@ -56,6 +58,8 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl int write(const char * start, size_t size) const { int bytes_written = hdfsWrite(fs.get(), fout, start, size); + if (write_settings.remote_throttler) + write_settings.remote_throttler->add(bytes_written); if (bytes_written < 0) throw Exception("Fail to write HDFS file: " + hdfs_uri + " " + std::string(hdfsGetLastError()), @@ -77,10 +81,11 @@ WriteBufferFromHDFS::WriteBufferFromHDFS( const std::string & hdfs_name_, const Poco::Util::AbstractConfiguration & config_, int replication_, + const WriteSettings & write_settings_, size_t buf_size_, int flags_) : BufferWithOwnMemory(buf_size_) - , impl(std::make_unique(hdfs_name_, config_, replication_, flags_)) + , impl(std::make_unique(hdfs_name_, config_, replication_, write_settings_, flags_)) { } diff --git a/src/Storages/HDFS/WriteBufferFromHDFS.h b/src/Storages/HDFS/WriteBufferFromHDFS.h index fe9af7dfba4..3cc11a35186 100644 --- a/src/Storages/HDFS/WriteBufferFromHDFS.h +++ b/src/Storages/HDFS/WriteBufferFromHDFS.h @@ -4,6 +4,7 @@ #if USE_HDFS #include +#include #include #include #include @@ -24,6 +25,7 @@ public: const String & hdfs_name_, const Poco::Util::AbstractConfiguration & config_, int replication_, + const WriteSettings & write_settings_ = {}, size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE, int flags = O_WRONLY); diff --git a/src/Storages/Hive/HiveFile.cpp b/src/Storages/Hive/HiveFile.cpp index 57acbdd577b..09c3aff4455 100644 --- a/src/Storages/Hive/HiveFile.cpp +++ b/src/Storages/Hive/HiveFile.cpp @@ -147,7 +147,7 @@ Range HiveORCFile::buildRange(const orc::ColumnStatistics * col_stats) void HiveORCFile::prepareReader() { - in = std::make_unique(namenode_url, path, getContext()->getGlobalContext()->getConfigRef()); + in = std::make_unique(namenode_url, path, getContext()->getGlobalContext()->getConfigRef(), getContext()->getReadSettings()); auto format_settings = getFormatSettings(getContext()); std::atomic is_stopped{0}; auto result = arrow::adapters::orc::ORCFileReader::Open(asArrowFile(*in, format_settings, is_stopped, "ORC", ORC_MAGIC_BYTES), arrow::default_memory_pool()); @@ -267,7 +267,7 @@ bool HiveParquetFile::useSplitMinMaxIndex() const void HiveParquetFile::prepareReader() { - in = std::make_unique(namenode_url, path, getContext()->getGlobalContext()->getConfigRef()); + in = std::make_unique(namenode_url, path, getContext()->getGlobalContext()->getConfigRef(), getContext()->getReadSettings()); auto format_settings = getFormatSettings(getContext()); std::atomic is_stopped{0}; THROW_ARROW_NOT_OK(parquet::arrow::OpenFile(asArrowFile(*in, format_settings, is_stopped, "Parquet", PARQUET_MAGIC_BYTES), arrow::default_memory_pool(), &reader)); diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index cea05aee0d5..ddd9f526091 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -218,7 +218,10 @@ public: auto get_raw_read_buf = [&]() -> std::unique_ptr { auto buf = std::make_unique( - hdfs_namenode_url, current_path, getContext()->getGlobalContext()->getConfigRef()); + hdfs_namenode_url, + current_path, + getContext()->getGlobalContext()->getConfigRef(), + getContext()->getReadSettings()); bool thread_pool_read = read_settings.remote_fs_method == RemoteFSReadMethod::threadpool; if (thread_pool_read) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 40f23fe5294..daf31698aad 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -2050,7 +2050,7 @@ bool KeyCondition::mayBeTrueInRange( } String KeyCondition::RPNElement::toString() const { return toString("column " + std::to_string(key_column), false); } -String KeyCondition::RPNElement::toString(const std::string_view & column_name, bool print_constants) const +String KeyCondition::RPNElement::toString(std::string_view column_name, bool print_constants) const { auto print_wrapped_column = [this, &column_name, print_constants](WriteBuffer & buf) { diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index 9a8719afa19..af85a90dd62 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -320,7 +320,7 @@ private: : function(function_), range(range_), key_column(key_column_) {} String toString() const; - String toString(const std::string_view & column_name, bool print_constants) const; + String toString(std::string_view column_name, bool print_constants) const; Function function = FUNCTION_UNKNOWN; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 5900ea0fdb7..64aaa40bd4c 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6229,8 +6229,13 @@ try part_log_elem.event_type = type; if (part_log_elem.event_type == PartLogElement::MERGE_PARTS) + { if (merge_entry) + { part_log_elem.merge_reason = PartLogElement::getMergeReasonType((*merge_entry)->merge_type); + part_log_elem.merge_algorithm = PartLogElement::getMergeAlgorithm((*merge_entry)->merge_algorithm); + } + } part_log_elem.error = static_cast(execution_status.code); part_log_elem.exception = execution_status.message; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index f6c80baba05..7967726edca 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1102,9 +1102,8 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry if (future_part.isDisjoint(result_part)) continue; - /// Parts are not disjoint, so new_part_name either contains or covers future_part. - if (!(future_part.contains(result_part) || result_part.contains(future_part))) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected non-disjoint parts: {} and {}", future_part_elem.first, new_part_name); + /// Parts are not disjoint. They can be even intersecting and it's not a problem, + /// because we may have two queue entries producing intersecting parts if there's DROP_RANGE between them (so virtual_parts are ok). /// We cannot execute `entry` (or upgrade its actual_part_name to `new_part_name`) /// while any covered or covering parts are processed. diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 1f2523c8645..dce45b2431a 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -15,6 +15,7 @@ #include #include #include +#include namespace DB { @@ -86,6 +87,62 @@ void StorageMongoDB::connectIfNotConnected() } +class StorageMongoDBSink : public SinkToStorage +{ +public: + explicit StorageMongoDBSink( + const std::string & collection_name_, + const std::string & db_name_, + const StorageMetadataPtr & metadata_snapshot_, + std::shared_ptr connection_) + : SinkToStorage(metadata_snapshot_->getSampleBlock()) + , collection_name(collection_name_) + , db_name(db_name_) + , metadata_snapshot{metadata_snapshot_} + , connection(connection_) + { + } + + String getName() const override { return "StorageMongoDBSink"; } + + void consume(Chunk chunk) override + { + Poco::MongoDB::Database db(db_name); + Poco::MongoDB::Document::Ptr index = new Poco::MongoDB::Document(); + + auto block = getHeader().cloneWithColumns(chunk.detachColumns()); + + size_t num_rows = block.rows(); + size_t num_cols = block.columns(); + + const auto columns = block.getColumns(); + const auto data_types = block.getDataTypes(); + const auto data_names = block.getNames(); + + std::vector row(num_cols); + for (const auto i : collections::range(0, num_rows)) + { + for (const auto j : collections::range(0, num_cols)) + { + WriteBufferFromOwnString ostr; + data_types[j]->getDefaultSerialization()->serializeText(*columns[j], i, ostr, FormatSettings{}); + row[j] = ostr.str(); + index->add(data_names[j], row[j]); + } + } + Poco::SharedPtr insert_request = db.createInsertRequest(collection_name); + insert_request->documents().push_back(index); + connection->sendRequest(*insert_request); + } + +private: + String collection_name; + String db_name; + StorageMetadataPtr metadata_snapshot; + std::shared_ptr connection; +}; + + Pipe StorageMongoDB::read( const Names & column_names, const StorageSnapshotPtr & storage_snapshot, @@ -109,6 +166,11 @@ Pipe StorageMongoDB::read( return Pipe(std::make_shared(connection, createCursor(database_name, collection_name, sample_block), sample_block, max_block_size)); } +SinkToStoragePtr StorageMongoDB::write(const ASTPtr & /* query */, const StorageMetadataPtr & metadata_snapshot, ContextPtr /* context */) +{ + connectIfNotConnected(); + return std::make_shared(collection_name, database_name, metadata_snapshot, connection); +} StorageMongoDBConfiguration StorageMongoDB::getConfiguration(ASTs engine_args, ContextPtr context) { diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h index cb0654433bc..0e00b80432b 100644 --- a/src/Storages/StorageMongoDB.h +++ b/src/Storages/StorageMongoDB.h @@ -39,6 +39,11 @@ public: size_t max_block_size, unsigned num_streams) override; + SinkToStoragePtr write( + const ASTPtr & query, + const StorageMetadataPtr & /*metadata_snapshot*/, + ContextPtr context) override; + static StorageMongoDBConfiguration getConfiguration(ASTs engine_args, ContextPtr context); private: diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index bed21a9affc..130bc75a65c 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -587,7 +587,8 @@ public: s3_configuration_.rw_settings, std::nullopt, DBMS_DEFAULT_BUFFER_SIZE, - threadPoolCallbackRunner(IOThreadPool::get())), + threadPoolCallbackRunner(IOThreadPool::get()), + context->getWriteSettings()), compression_method, 3); writer diff --git a/src/Storages/System/StorageSystemDisks.cpp b/src/Storages/System/StorageSystemDisks.cpp index c84a4f17510..86238ab8df1 100644 --- a/src/Storages/System/StorageSystemDisks.cpp +++ b/src/Storages/System/StorageSystemDisks.cpp @@ -57,7 +57,7 @@ Pipe StorageSystemDisks::read( col_type->insert(toString(disk_ptr->getType())); String cache_path; - if (disk_ptr->isRemote()) + if (disk_ptr->supportsCache()) cache_path = disk_ptr->getCacheBasePath(); col_cache_path->insert(cache_path); diff --git a/src/Storages/System/StorageSystemPrivileges.cpp b/src/Storages/System/StorageSystemPrivileges.cpp index 8cf1accfe34..70163979f72 100644 --- a/src/Storages/System/StorageSystemPrivileges.cpp +++ b/src/Storages/System/StorageSystemPrivileges.cpp @@ -85,7 +85,7 @@ void StorageSystemPrivileges::fillData(MutableColumns & res_columns, ContextPtr, auto & column_parent_group = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()).getData(); auto & column_parent_group_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); - auto add_row = [&](AccessType access_type, const std::string_view & aliases, Level max_level, AccessType parent_group) + auto add_row = [&](AccessType access_type, std::string_view aliases, Level max_level, AccessType parent_group) { column_access_type.push_back(static_cast(access_type)); diff --git a/src/Storages/System/StorageSystemRemoteDataPaths.cpp b/src/Storages/System/StorageSystemRemoteDataPaths.cpp index d39a0d2482d..a482f5d87ca 100644 --- a/src/Storages/System/StorageSystemRemoteDataPaths.cpp +++ b/src/Storages/System/StorageSystemRemoteDataPaths.cpp @@ -51,7 +51,7 @@ Pipe StorageSystemRemoteDataPaths::read( { if (disk->isRemote()) { - std::vector remote_paths_by_local_path; + std::vector remote_paths_by_local_path; disk->getRemotePathsRecursive("store", remote_paths_by_local_path); disk->getRemotePathsRecursive("data", remote_paths_by_local_path); @@ -60,19 +60,19 @@ Pipe StorageSystemRemoteDataPaths::read( if (!cache_base_path.empty()) cache = FileCacheFactory::instance().get(cache_base_path); - for (const auto & [local_path, remote_paths] : remote_paths_by_local_path) + for (const auto & [local_path, storage_objects] : remote_paths_by_local_path) { - for (const auto & remote_path : remote_paths) + for (const auto & object : storage_objects) { col_disk_name->insert(disk_name); col_base_path->insert(disk->getPath()); col_cache_base_path->insert(cache_base_path); col_local_path->insert(local_path); - col_remote_path->insert(remote_path.path); + col_remote_path->insert(object.absolute_path); if (cache) { - auto cache_paths = cache->tryGetCachePaths(cache->hash(remote_path.path)); + auto cache_paths = cache->tryGetCachePaths(cache->hash(object.getPathKeyForCache())); col_cache_paths->insert(Array(cache_paths.begin(), cache_paths.end())); } else diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp index 5d37aa5b08a..cdd04964f55 100644 --- a/src/Storages/System/StorageSystemStackTrace.cpp +++ b/src/Storages/System/StorageSystemStackTrace.cpp @@ -5,10 +5,14 @@ #include #include +#include #include #include +#include +#include +#include #include #include #include @@ -16,8 +20,11 @@ #include #include #include -#include +#include #include +#include +#include +#include namespace DB @@ -147,13 +154,84 @@ namespace throw Exception("Logical error: read wrong number of bytes from pipe", ErrorCodes::LOGICAL_ERROR); } } + + ColumnPtr getFilteredThreadIds(ASTPtr query, ContextPtr context) + { + MutableColumnPtr all_thread_ids = ColumnUInt64::create(); + + std::filesystem::directory_iterator end; + + /// There is no better way to enumerate threads in a process other than looking into procfs. + for (std::filesystem::directory_iterator it("/proc/self/task"); it != end; ++it) + { + pid_t tid = parse(it->path().filename()); + all_thread_ids->insert(tid); + } + + Block block { ColumnWithTypeAndName(std::move(all_thread_ids), std::make_shared(), "thread_id") }; + VirtualColumnUtils::filterBlockWithQuery(query, block, context); + return block.getByPosition(0).column; + } + + using ThreadIdToName = std::unordered_map>; + ThreadIdToName getFilteredThreadNames(ASTPtr query, ContextPtr context, const PaddedPODArray & thread_ids) + { + ThreadIdToName tid_to_name; + MutableColumnPtr all_thread_names = ColumnString::create(); + + for (UInt64 tid : thread_ids) + { + std::filesystem::path thread_name_path = fmt::format("/proc/self/task/{}/comm", tid); + String thread_name; + if (std::filesystem::exists(thread_name_path)) + { + constexpr size_t comm_buf_size = 32; /// More than enough for thread name + ReadBufferFromFile comm(thread_name_path.string(), comm_buf_size); + readEscapedStringUntilEOL(thread_name, comm); + comm.close(); + } + + tid_to_name[tid] = thread_name; + all_thread_names->insert(thread_name); + } + + Block block { ColumnWithTypeAndName(std::move(all_thread_names), std::make_shared(), "thread_name") }; + VirtualColumnUtils::filterBlockWithQuery(query, block, context); + ColumnPtr thread_names = std::move(block.getByPosition(0).column); + + std::unordered_set filtered_thread_names; + for (size_t i = 0; i != thread_names->size(); ++i) + { + const auto & thread_name = thread_names->getDataAt(i); + filtered_thread_names.emplace(thread_name); + } + + for (auto it = tid_to_name.begin(); it != tid_to_name.end();) + { + if (!filtered_thread_names.contains(it->second)) + it = tid_to_name.erase(it); + else + ++it; + } + + return tid_to_name; + } } StorageSystemStackTrace::StorageSystemStackTrace(const StorageID & table_id_) - : IStorageSystemOneBlock(table_id_) + : IStorage(table_id_) , log(&Poco::Logger::get("StorageSystemStackTrace")) { + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(ColumnsDescription({ + { "thread_name", std::make_shared() }, + { "thread_id", std::make_shared() }, + { "query_id", std::make_shared() }, + { "trace", std::make_shared(std::make_shared()) }, + }, { /* aliases */ })); + setInMemoryMetadata(storage_metadata); + notification_pipe.open(); /// Setup signal handler. @@ -173,23 +251,40 @@ StorageSystemStackTrace::StorageSystemStackTrace(const StorageID & table_id_) } -NamesAndTypesList StorageSystemStackTrace::getNamesAndTypes() +Pipe StorageSystemStackTrace::read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum /*processed_stage*/, + const size_t /*max_block_size*/, + const unsigned /*num_streams*/) { - return - { - { "thread_name", std::make_shared() }, - { "thread_id", std::make_shared() }, - { "query_id", std::make_shared() }, - { "trace", std::make_shared(std::make_shared()) } - }; -} + storage_snapshot->check(column_names); - -void StorageSystemStackTrace::fillData(MutableColumns & res_columns, ContextPtr, const SelectQueryInfo &) const -{ /// It shouldn't be possible to do concurrent reads from this table. std::lock_guard lock(mutex); + /// Create a mask of what columns are needed in the result. + + NameSet names_set(column_names.begin(), column_names.end()); + + Block sample_block = storage_snapshot->metadata->getSampleBlock(); + + std::vector columns_mask(sample_block.columns()); + for (size_t i = 0, size = columns_mask.size(); i < size; ++i) + { + if (names_set.contains(sample_block.getByPosition(i).name)) + { + columns_mask[i] = 1; + } + } + + bool send_signal = names_set.contains("trace") || names_set.contains("query_id"); + bool read_thread_names = names_set.contains("thread_name"); + + MutableColumns res_columns = sample_block.cloneEmptyColumns(); + /// Send a signal to every thread and wait for result. /// We must wait for every thread one by one sequentially, /// because there is a limit on number of queued signals in OS and otherwise signals may get lost. @@ -197,71 +292,85 @@ void StorageSystemStackTrace::fillData(MutableColumns & res_columns, ContextPtr, /// Obviously, results for different threads may be out of sync. - /// There is no better way to enumerate threads in a process other than looking into procfs. + ColumnPtr thread_ids = getFilteredThreadIds(query_info.query, context); + const auto & thread_ids_data = assert_cast(*thread_ids).getData(); - std::filesystem::directory_iterator end; - for (std::filesystem::directory_iterator it("/proc/self/task"); it != end; ++it) + ThreadIdToName thread_names; + if (read_thread_names) + thread_names = getFilteredThreadNames(query_info.query, context, thread_ids_data); + + for (UInt64 tid : thread_ids_data) { - pid_t tid = parse(it->path().filename()); - - sigval sig_value{}; - sig_value.sival_int = sequence_num.load(std::memory_order_acquire); - if (0 != ::sigqueue(tid, sig, sig_value)) - { - /// The thread may has been already finished. - if (ESRCH == errno) - continue; - - throwFromErrno("Cannot send signal with sigqueue", ErrorCodes::CANNOT_SIGQUEUE); - } - - std::filesystem::path thread_name_path = it->path(); - thread_name_path.append("comm"); + size_t res_index = 0; String thread_name; - if (std::filesystem::exists(thread_name_path)) + if (read_thread_names) { - constexpr size_t comm_buf_size = 32; /// More than enough for thread name - ReadBufferFromFile comm(thread_name_path.string(), comm_buf_size); - readEscapedStringUntilEOL(thread_name, comm); - comm.close(); + if (auto it = thread_names.find(tid); it != thread_names.end()) + thread_name = it->second; + else + continue; /// was filtered out by "thread_name" condition } - /// Just in case we will wait for pipe with timeout. In case signal didn't get processed. - - if (wait(100) && sig_value.sival_int == data_ready_num.load(std::memory_order_acquire)) + if (!send_signal) { - size_t stack_trace_size = stack_trace.getSize(); - size_t stack_trace_offset = stack_trace.getOffset(); - - Array arr; - arr.reserve(stack_trace_size - stack_trace_offset); - for (size_t i = stack_trace_offset; i < stack_trace_size; ++i) - arr.emplace_back(reinterpret_cast(stack_trace.getFramePointers()[i])); - - res_columns[0]->insert(thread_name); - res_columns[1]->insert(tid); - res_columns[2]->insertData(query_id_data, query_id_size); - res_columns[3]->insert(arr); + res_columns[res_index++]->insert(thread_name); + res_columns[res_index++]->insert(tid); + res_columns[res_index++]->insertDefault(); + res_columns[res_index++]->insertDefault(); } else { - LOG_DEBUG(log, "Cannot obtain a stack trace for thread {}", tid); + sigval sig_value{}; - /// Cannot obtain a stack trace. But create a record in result nevertheless. + sig_value.sival_int = sequence_num.load(std::memory_order_acquire); + if (0 != ::sigqueue(tid, sig, sig_value)) + { + /// The thread may has been already finished. + if (ESRCH == errno) + continue; - res_columns[0]->insert(thread_name); - res_columns[1]->insert(tid); - res_columns[2]->insertDefault(); - res_columns[3]->insertDefault(); + throwFromErrno("Cannot send signal with sigqueue", ErrorCodes::CANNOT_SIGQUEUE); + } + + /// Just in case we will wait for pipe with timeout. In case signal didn't get processed. + if (send_signal && wait(100) && sig_value.sival_int == data_ready_num.load(std::memory_order_acquire)) + { + size_t stack_trace_size = stack_trace.getSize(); + size_t stack_trace_offset = stack_trace.getOffset(); + + Array arr; + arr.reserve(stack_trace_size - stack_trace_offset); + for (size_t i = stack_trace_offset; i < stack_trace_size; ++i) + arr.emplace_back(reinterpret_cast(stack_trace.getFramePointers()[i])); + + res_columns[res_index++]->insert(thread_name); + res_columns[res_index++]->insert(tid); + res_columns[res_index++]->insertData(query_id_data, query_id_size); + res_columns[res_index++]->insert(arr); + } + else + { + LOG_DEBUG(log, "Cannot obtain a stack trace for thread {}", tid); + + res_columns[res_index++]->insert(thread_name); + res_columns[res_index++]->insert(tid); + res_columns[res_index++]->insertDefault(); + res_columns[res_index++]->insertDefault(); + } + + /// Signed integer overflow is undefined behavior in both C and C++. However, according to + /// C++ standard, Atomic signed integer arithmetic is defined to use two's complement; there + /// are no undefined results. See https://en.cppreference.com/w/cpp/atomic/atomic and + /// http://eel.is/c++draft/atomics.types.generic#atomics.types.int-8 + ++sequence_num; } - - /// Signed integer overflow is undefined behavior in both C and C++. However, according to - /// C++ standard, Atomic signed integer arithmetic is defined to use two's complement; there - /// are no undefined results. See https://en.cppreference.com/w/cpp/atomic/atomic and - /// http://eel.is/c++draft/atomics.types.generic#atomics.types.int-8 - ++sequence_num; } + + UInt64 num_rows = res_columns.at(0)->size(); + Chunk chunk(std::move(res_columns), num_rows); + + return Pipe(std::make_shared(sample_block, std::move(chunk))); } } diff --git a/src/Storages/System/StorageSystemStackTrace.h b/src/Storages/System/StorageSystemStackTrace.h index c039ae53170..dd613882e49 100644 --- a/src/Storages/System/StorageSystemStackTrace.h +++ b/src/Storages/System/StorageSystemStackTrace.h @@ -3,7 +3,7 @@ #ifdef OS_LINUX /// Because of 'sigqueue' functions and RT signals. #include -#include +#include namespace Poco { @@ -19,20 +19,26 @@ class Context; /// Allows to introspect stack trace of all server threads. /// It acts like an embedded debugger. /// More than one instance of this table cannot be used. -class StorageSystemStackTrace final : public IStorageSystemOneBlock +class StorageSystemStackTrace final : public IStorage { public: explicit StorageSystemStackTrace(const StorageID & table_id_); String getName() const override { return "SystemStackTrace"; } - static NamesAndTypesList getNamesAndTypes(); + + Pipe read( + const Names & column_names, + const StorageSnapshotPtr & storage_snapshot, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + + bool isSystemStorage() const override { return true; } protected: - using IStorageSystemOneBlock::IStorageSystemOneBlock; - void fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo & query_info) const override; - mutable std::mutex mutex; - Poco::Logger * log; }; diff --git a/src/TableFunctions/TableFunctionMongoDB.cpp b/src/TableFunctions/TableFunctionMongoDB.cpp new file mode 100644 index 00000000000..5e96b85e64c --- /dev/null +++ b/src/TableFunctions/TableFunctionMongoDB.cpp @@ -0,0 +1,104 @@ +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + + +StoragePtr TableFunctionMongoDB::executeImpl(const ASTPtr & /*ast_function*/, + ContextPtr context, const String & table_name, ColumnsDescription /*cached_columns*/) const +{ + auto columns = getActualTableStructure(context); + auto storage = std::make_shared( + StorageID(configuration->database, table_name), + configuration->host, + configuration->port, + configuration->database, + configuration->table, + configuration->username, + configuration->password, + configuration->options, + columns, + ConstraintsDescription(), + String{}); + storage->startup(); + return storage; +} + +ColumnsDescription TableFunctionMongoDB::getActualTableStructure(ContextPtr context) const +{ + return parseColumnsListFromString(structure, context); +} + +void TableFunctionMongoDB::parseArguments(const ASTPtr & ast_function, ContextPtr context) +{ + const auto & func_args = ast_function->as(); + if (!func_args.arguments) + throw Exception("Table function 'mongodb' must have arguments.", ErrorCodes::BAD_ARGUMENTS); + + ASTs & args = func_args.arguments->children; + + if (args.size() < 6 || args.size() > 7) + { + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Table function 'mongodb' requires from 6 to 7 parameters: mongodb('host:port', database, collection, 'user', 'password', structure, [, 'options'])"); + } + + ASTs main_arguments(args.begin(), args.begin() + 5); + + for (size_t i = 5; i < args.size(); ++i) + { + if (const auto * ast_func = typeid_cast(args[i].get())) + { + const auto * args_expr = assert_cast(ast_func->arguments.get()); + auto function_args = args_expr->children; + if (function_args.size() != 2) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument"); + + auto arg_name = function_args[0]->as()->name(); + + if (arg_name == "structure") + structure = checkAndGetLiteralArgument(function_args[1], "structure"); + else if (arg_name == "options") + main_arguments.push_back(function_args[1]); + } + else if (i == 5) + { + structure = checkAndGetLiteralArgument(args[i], "structure"); + } + else if (i == 6) + { + main_arguments.push_back(args[i]); + } + } + + configuration = StorageMongoDB::getConfiguration(main_arguments, context); +} + + +void registerTableFunctionMongoDB(TableFunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/TableFunctions/TableFunctionMongoDB.h b/src/TableFunctions/TableFunctionMongoDB.h new file mode 100644 index 00000000000..40e4802e9e6 --- /dev/null +++ b/src/TableFunctions/TableFunctionMongoDB.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class TableFunctionMongoDB : public ITableFunction +{ +public: + static constexpr auto name = "mongodb"; + + std::string getName() const override { return name; } + +private: + StoragePtr executeImpl( + const ASTPtr & ast_function, ContextPtr context, + const std::string & table_name, ColumnsDescription cached_columns) const override; + + const char * getStorageTypeName() const override { return "MongoDB"; } + + ColumnsDescription getActualTableStructure(ContextPtr context) const override; + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; + + std::optional configuration; + String structure; +}; + +} diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp index e9d2fa18639..12ca4abe113 100644 --- a/src/TableFunctions/registerTableFunctions.cpp +++ b/src/TableFunctions/registerTableFunctions.cpp @@ -19,6 +19,7 @@ void registerTableFunctions() registerTableFunctionValues(factory); registerTableFunctionInput(factory); registerTableFunctionGenerate(factory); + registerTableFunctionMongoDB(factory); registerTableFunctionMeiliSearch(factory); diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index 906cf7a74b9..49a1ef60a6b 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -17,6 +17,7 @@ void registerTableFunctionURL(TableFunctionFactory & factory); void registerTableFunctionValues(TableFunctionFactory & factory); void registerTableFunctionInput(TableFunctionFactory & factory); void registerTableFunctionGenerate(TableFunctionFactory & factory); +void registerTableFunctionMongoDB(TableFunctionFactory & factory); void registerTableFunctionMeiliSearch(TableFunctionFactory & factory); diff --git a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml index a6e2d29c5d5..e414ae5a259 100644 --- a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml @@ -34,6 +34,16 @@ 33554432 1 + + s3 + http://minio1:9001/root/data/ + minio + minio123 + 33554432 + 1 + /jbod1/ + 1000000000 + @@ -67,6 +77,13 @@ + + +

+ s3_with_cache_and_jbod +
+ + diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index 129adce47ca..3ce2a08ae74 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -26,6 +26,18 @@ def cluster(): ], with_minio=True, ) + + cluster.add_instance( + "node_with_limited_disk", + main_configs=[ + "configs/config.d/storage_conf.xml", + "configs/config.d/bg_processing_pool_conf.xml", + ], + with_minio=True, + tmpfs=[ + "/jbod1:size=2M", + ], + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -678,3 +690,22 @@ def test_lazy_seek_optimization_for_async_read(cluster, node_name): minio = cluster.minio_client for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) + + +@pytest.mark.parametrize("node_name", ["node_with_limited_disk"]) +def test_cache_with_full_disk_space(cluster, node_name): + node = cluster.instances[node_name] + node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node.query( + "CREATE TABLE s3_test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_with_cache_and_jbod';" + ) + node.query( + "INSERT INTO s3_test SELECT * FROM generateRandom('key UInt32, value String') LIMIT 500000" + ) + node.query( + "SELECT * FROM s3_test WHERE value LIKE '%abc%' ORDER BY value FORMAT Null" + ) + assert node.contains_in_log( + "Insert into cache is skipped due to insufficient disk space" + ) + node.query("DROP TABLE IF EXISTS s3_test NO DELAY") diff --git a/tests/integration/test_storage_mongodb/test.py b/tests/integration/test_storage_mongodb/test.py index d8ca207d0a6..74b2b15fda0 100644 --- a/tests/integration/test_storage_mongodb/test.py +++ b/tests/integration/test_storage_mongodb/test.py @@ -253,3 +253,30 @@ def test_missing_columns(started_cluster): result = node.query("SELECT count() FROM simple_mongo_table WHERE isNull(data)") assert result == "10\n" simple_mongo_table.drop() + + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) +def test_simple_insert_select(started_cluster): + mongo_connection = get_mongo_connection(started_cluster) + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + simple_mongo_table = db["simple_table"] + + node = started_cluster.instances["node"] + node.query("DROP TABLE IF EXISTS simple_mongo_table") + node.query( + "CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')" + ) + node.query("INSERT INTO simple_mongo_table SELECT 1, 'kek'") + + assert ( + node.query("SELECT data from simple_mongo_table where key = 1").strip() == "kek" + ) + node.query("INSERT INTO simple_mongo_table(key) SELECT 12") + assert int(node.query("SELECT count() from simple_mongo_table")) == 2 + assert ( + node.query("SELECT data from simple_mongo_table where key = 12").strip() == "" + ) + + node.query("DROP TABLE simple_mongo_table") + simple_mongo_table.drop() diff --git a/tests/integration/test_table_function_mongodb/__init__.py b/tests/integration/test_table_function_mongodb/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_table_function_mongodb/configs_secure/config.d/ssl_conf.xml b/tests/integration/test_table_function_mongodb/configs_secure/config.d/ssl_conf.xml new file mode 100644 index 00000000000..3efe98e7045 --- /dev/null +++ b/tests/integration/test_table_function_mongodb/configs_secure/config.d/ssl_conf.xml @@ -0,0 +1,8 @@ + + + + + none + + + diff --git a/tests/integration/test_table_function_mongodb/test.py b/tests/integration/test_table_function_mongodb/test.py new file mode 100644 index 00000000000..e0ad71b0079 --- /dev/null +++ b/tests/integration/test_table_function_mongodb/test.py @@ -0,0 +1,276 @@ +import pymongo + +import pytest +from helpers.client import QueryRuntimeException + +from helpers.cluster import ClickHouseCluster + + +@pytest.fixture(scope="module") +def started_cluster(request): + try: + cluster = ClickHouseCluster(__file__) + node = cluster.add_instance( + "node", + with_mongo=True, + main_configs=[ + "configs_secure/config.d/ssl_conf.xml", + ], + with_mongo_secure=request.param, + ) + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def get_mongo_connection(started_cluster, secure=False, with_credentials=True): + connection_str = "" + if with_credentials: + connection_str = "mongodb://root:clickhouse@localhost:{}".format( + started_cluster.mongo_port + ) + else: + connection_str = "mongodb://localhost:{}".format( + started_cluster.mongo_no_cred_port + ) + if secure: + connection_str += "/?tls=true&tlsAllowInvalidCertificates=true" + return pymongo.MongoClient(connection_str) + + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) +def test_simple_select(started_cluster): + mongo_connection = get_mongo_connection(started_cluster) + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + simple_mongo_table = db["simple_table"] + + node = started_cluster.instances["node"] + for i in range(0, 100): + node.query( + "INSERT INTO FUNCTION mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data String') (key, data) VALUES ({}, '{}')".format( + i, hex(i * i) + ) + ) + assert ( + node.query( + "SELECT COUNT() FROM mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data String')" + ) + == "100\n" + ) + assert ( + node.query( + "SELECT sum(key) FROM mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data String')" + ) + == str(sum(range(0, 100))) + "\n" + ) + assert ( + node.query( + "SELECT sum(key) FROM mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', 'key UInt64, data String')" + ) + == str(sum(range(0, 100))) + "\n" + ) + + assert ( + node.query( + "SELECT data from mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data String') where key = 42" + ) + == hex(42 * 42) + "\n" + ) + simple_mongo_table.drop() + + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) +def test_complex_data_type(started_cluster): + mongo_connection = get_mongo_connection(started_cluster) + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + incomplete_mongo_table = db["complex_table"] + data = [] + for i in range(0, 100): + data.append({"key": i, "data": hex(i * i), "dict": {"a": i, "b": str(i)}}) + incomplete_mongo_table.insert_many(data) + + node = started_cluster.instances["node"] + + assert ( + node.query( + "SELECT COUNT() FROM mongodb('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse', structure='key UInt64, data String, dict Map(UInt64, String)')" + ) + == "100\n" + ) + assert ( + node.query( + "SELECT sum(key) FROM mongodb('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse', structure='key UInt64, data String, dict Map(UInt64, String)')" + ) + == str(sum(range(0, 100))) + "\n" + ) + + assert ( + node.query( + "SELECT data from mongodb('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse', structure='key UInt64, data String, dict Map(UInt64, String)') where key = 42" + ) + == hex(42 * 42) + "\n" + ) + incomplete_mongo_table.drop() + + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) +def test_incorrect_data_type(started_cluster): + mongo_connection = get_mongo_connection(started_cluster) + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + strange_mongo_table = db["strange_table"] + data = [] + for i in range(0, 100): + data.append({"key": i, "data": hex(i * i), "aaaa": "Hello"}) + strange_mongo_table.insert_many(data) + + node = started_cluster.instances["node"] + + with pytest.raises(QueryRuntimeException): + node.query( + "SELECT aaaa FROM mongodb('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse', structure='key UInt64, data String')" + ) + + strange_mongo_table.drop() + + +@pytest.mark.parametrize("started_cluster", [True], indirect=["started_cluster"]) +def test_secure_connection(started_cluster): + mongo_connection = get_mongo_connection(started_cluster, secure=True) + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + simple_mongo_table = db["simple_table"] + data = [] + for i in range(0, 100): + data.append({"key": i, "data": hex(i * i)}) + simple_mongo_table.insert_many(data) + + node = started_cluster.instances["node"] + + assert ( + node.query( + "SELECT COUNT() FROM mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data String', options='ssl=true')" + ) + == "100\n" + ) + assert ( + node.query( + "SELECT sum(key) FROM mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data String', options='ssl=true')" + ) + == str(sum(range(0, 100))) + "\n" + ) + assert ( + node.query( + "SELECT sum(key) FROM mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', 'key UInt64, data String', 'ssl=true')" + ) + == str(sum(range(0, 100))) + "\n" + ) + + assert ( + node.query( + "SELECT data from mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data String', options='ssl=true') where key = 42" + ) + == hex(42 * 42) + "\n" + ) + simple_mongo_table.drop() + + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) +def test_predefined_connection_configuration(started_cluster): + mongo_connection = get_mongo_connection(started_cluster) + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + simple_mongo_table = db["simple_table"] + data = [] + for i in range(0, 100): + data.append({"key": i, "data": hex(i * i)}) + simple_mongo_table.insert_many(data) + + node = started_cluster.instances["node"] + assert ( + node.query( + "SELECT count() FROM mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data String')" + ) + == "100\n" + ) + simple_mongo_table.drop() + + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) +def test_no_credentials(started_cluster): + mongo_connection = get_mongo_connection(started_cluster, with_credentials=False) + db = mongo_connection["test"] + simple_mongo_table = db["simple_table"] + data = [] + for i in range(0, 100): + data.append({"key": i, "data": hex(i * i)}) + simple_mongo_table.insert_many(data) + + node = started_cluster.instances["node"] + assert ( + node.query( + "SELECT count() FROM mongodb('mongo2:27017', 'test', 'simple_table', '', '', structure='key UInt64, data String')" + ) + == "100\n" + ) + simple_mongo_table.drop() + + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) +def test_auth_source(started_cluster): + mongo_connection = get_mongo_connection(started_cluster, with_credentials=False) + admin_db = mongo_connection["admin"] + admin_db.add_user( + "root", + "clickhouse", + roles=[{"role": "userAdminAnyDatabase", "db": "admin"}, "readWriteAnyDatabase"], + ) + simple_mongo_table = admin_db["simple_table"] + data = [] + for i in range(0, 50): + data.append({"key": i, "data": hex(i * i)}) + simple_mongo_table.insert_many(data) + db = mongo_connection["test"] + simple_mongo_table = db["simple_table"] + data = [] + for i in range(0, 100): + data.append({"key": i, "data": hex(i * i)}) + simple_mongo_table.insert_many(data) + + node = started_cluster.instances["node"] + + node.query_and_get_error( + "SELECT count() FROM mongodb('mongo2:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data String')" + ) + + assert ( + node.query( + "SELECT count() FROM mongodb('mongo2:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data String', options='authSource=admin')" + ) + == "100\n" + ) + simple_mongo_table.drop() + + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) +def test_missing_columns(started_cluster): + mongo_connection = get_mongo_connection(started_cluster) + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + simple_mongo_table = db["simple_table"] + data = [] + for i in range(0, 10): + data.append({"key": i, "data": hex(i * i)}) + for i in range(0, 10): + data.append({"key": i}) + simple_mongo_table.insert_many(data) + + node = started_cluster.instances["node"] + result = node.query( + "SELECT count() FROM mongodb('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', structure='key UInt64, data Nullable(String)') WHERE isNull(data)" + ) + assert result == "10\n" + simple_mongo_table.drop() diff --git a/tests/queries/0_stateless/00050_any_left_join.sql b/tests/queries/0_stateless/00050_any_left_join.sql index 61ccb200307..5019de9fce7 100644 --- a/tests/queries/0_stateless/00050_any_left_join.sql +++ b/tests/queries/0_stateless/00050_any_left_join.sql @@ -6,4 +6,5 @@ ANY LEFT JOIN ( SELECT number * 2 AS k, number AS joined FROM system.numbers LIMIT 10 ) AS b -USING k; +USING k +ORDER BY k; diff --git a/tests/queries/0_stateless/00053_all_inner_join.sql b/tests/queries/0_stateless/00053_all_inner_join.sql index b2e65c80574..f4f84069d05 100644 --- a/tests/queries/0_stateless/00053_all_inner_join.sql +++ b/tests/queries/0_stateless/00053_all_inner_join.sql @@ -6,4 +6,5 @@ ALL INNER JOIN ( SELECT intDiv(number, 2) AS k, number AS joined FROM system.numbers LIMIT 10 ) AS b -USING k; +USING k +ORDER BY k, joined; diff --git a/tests/queries/0_stateless/00054_join_string.sql b/tests/queries/0_stateless/00054_join_string.sql index 274b19dc075..0e7a5520e78 100644 --- a/tests/queries/0_stateless/00054_join_string.sql +++ b/tests/queries/0_stateless/00054_join_string.sql @@ -6,4 +6,6 @@ ALL LEFT JOIN ( SELECT reinterpretAsString(intDiv(number, 2) + reinterpretAsUInt8('A')) AS k, number AS joined FROM system.numbers LIMIT 10 ) js2 -USING k; +USING k +ORDER BY k, joined +; diff --git a/tests/queries/0_stateless/00057_join_aliases.sql b/tests/queries/0_stateless/00057_join_aliases.sql index 6568e36fbda..481b0621ed7 100644 --- a/tests/queries/0_stateless/00057_join_aliases.sql +++ b/tests/queries/0_stateless/00057_join_aliases.sql @@ -1 +1,6 @@ -SELECT number, number / 2 AS n, j1, j2 FROM system.numbers ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 10) js2 USING n LIMIT 10 +SELECT * FROM ( + SELECT number, number / 2 AS n, j1, j2 + FROM system.numbers + ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 10) js2 + USING n LIMIT 10 +) ORDER BY n; diff --git a/tests/queries/0_stateless/00163_shard_join_with_empty_table.sql b/tests/queries/0_stateless/00163_shard_join_with_empty_table.sql index 71c570cf762..777b2443416 100644 --- a/tests/queries/0_stateless/00163_shard_join_with_empty_table.sql +++ b/tests/queries/0_stateless/00163_shard_join_with_empty_table.sql @@ -3,7 +3,30 @@ SET any_join_distinct_right_table_keys = 1; SET joined_subquery_requires_alias = 0; -SELECT number, number / 2 AS n, j1, j2 FROM remote('127.0.0.{2,3}', system.numbers) ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) USING n LIMIT 10; -SELECT dummy + 2 AS number, number / 2 AS n, j1, j2 FROM remote('127.0.0.{2,3}', system.one) ANY INNER JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) USING n LIMIT 10; -SELECT number, number / 2 AS n, j1, j2 FROM remote('127.0.0.{2,3}', system.numbers) GLOBAL ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) USING n LIMIT 10; -SELECT dummy + 2 AS number, number / 2 AS n, j1, j2 FROM remote('127.0.0.{2,3}', system.one) GLOBAL ANY INNER JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) USING n LIMIT 10; +SELECT * FROM ( + SELECT number, number / 2 AS n, j1, j2 + FROM remote('127.0.0.{2,3}', system.numbers) + ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) + USING n LIMIT 10 +) ORDER BY number; + +SELECT * FROM ( + SELECT dummy + 2 AS number, number / 2 AS n, j1, j2 + FROM remote('127.0.0.{2,3}', system.one) + ANY INNER JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) + USING n LIMIT 10 +) ORDER BY number; + +SELECT * FROM ( + SELECT number, number / 2 AS n, j1, j2 + FROM remote('127.0.0.{2,3}', system.numbers) + GLOBAL ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) + USING n LIMIT 10 +) ORDER BY number; + +SELECT * FROM ( + SELECT dummy + 2 AS number, number / 2 AS n, j1, j2 + FROM remote('127.0.0.{2,3}', system.one) + GLOBAL ANY INNER JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) + USING n LIMIT 10 +) ORDER BY number; diff --git a/tests/queries/0_stateless/00169_join_constant_keys.sql b/tests/queries/0_stateless/00169_join_constant_keys.sql index 16ab696c0f3..03c01c074f7 100644 --- a/tests/queries/0_stateless/00169_join_constant_keys.sql +++ b/tests/queries/0_stateless/00169_join_constant_keys.sql @@ -14,4 +14,5 @@ FROM arrayJoin([1, 3, 2]) AS key1, 0 AS key2, 999 AS table_1 -) js2 USING key2, key1; +) js2 USING key2, key1 +ORDER BY key1; diff --git a/tests/queries/0_stateless/00689_join_table_function.sql b/tests/queries/0_stateless/00689_join_table_function.sql index d44d1009ce8..c8e9100dcc0 100644 --- a/tests/queries/0_stateless/00689_join_table_function.sql +++ b/tests/queries/0_stateless/00689_join_table_function.sql @@ -1 +1 @@ -SELECT * FROM numbers(3) AS a ANY LEFT JOIN numbers(3) AS b ON a.number = b.number +SELECT * FROM numbers(3) AS a ANY LEFT JOIN numbers(3) AS b ON a.number = b.number ORDER BY a.number; diff --git a/tests/queries/0_stateless/00702_join_with_using.sql b/tests/queries/0_stateless/00702_join_with_using.sql index 24e57e1d555..4f81e757dfc 100644 --- a/tests/queries/0_stateless/00702_join_with_using.sql +++ b/tests/queries/0_stateless/00702_join_with_using.sql @@ -26,14 +26,14 @@ values ('1', 'John'), ('2', 'Jack'), ('3', 'Daniel'), ('4', 'James'), ('5', 'Ama insert into children (id, childName) values ('1', 'Robert'), ('1', 'Susan'), ('3', 'Sarah'), ('4', 'David'), ('4', 'Joseph'), ('5', 'Robert'); -select * from persons all inner join children using id; -select * from persons all inner join (select * from children) as j using id; -select * from (select * from persons) as s all inner join (select * from children ) as j using id; +select * from persons all inner join children using id order by id, name, childName; +select * from persons all inner join (select * from children) as j using id order by id, name, childName; +select * from (select * from persons) as s all inner join (select * from children ) as j using id order by id, name, childName; -- set joined_subquery_requires_alias = 0; -select * from persons all inner join (select * from children) using id; -select * from (select * from persons) all inner join (select * from children) using id; -select * from (select * from persons) as s all inner join (select * from children) using id; +select * from persons all inner join (select * from children) using id order by id, name, childName; +select * from (select * from persons) all inner join (select * from children) using id order by id, name, childName; +select * from (select * from persons) as s all inner join (select * from children) using id order by id, name, childName; drop table persons; drop table children; diff --git a/tests/queries/0_stateless/00725_join_on_bug_3.sql b/tests/queries/0_stateless/00725_join_on_bug_3.sql index 08b39d899cf..678516b4fc5 100644 --- a/tests/queries/0_stateless/00725_join_on_bug_3.sql +++ b/tests/queries/0_stateless/00725_join_on_bug_3.sql @@ -7,7 +7,7 @@ insert into t_00725_3 values(2,2); create table z_00725_3(c Int64, d Int64, e Int64) engine = TinyLog; insert into z_00725_3 values(1,1,1); -select * from t_00725_3 all left join z_00725_3 on (z_00725_3.c = t_00725_3.a and z_00725_3.d = t_00725_3.b); +select * from t_00725_3 all left join z_00725_3 on (z_00725_3.c = t_00725_3.a and z_00725_3.d = t_00725_3.b) ORDER BY t_00725_3.a; drop table if exists t_00725_3; drop table if exists z_00725_3; diff --git a/tests/queries/0_stateless/00818_join_bug_4271.sql b/tests/queries/0_stateless/00818_join_bug_4271.sql index ce11088fd95..7bf3b4bffd6 100644 --- a/tests/queries/0_stateless/00818_join_bug_4271.sql +++ b/tests/queries/0_stateless/00818_join_bug_4271.sql @@ -7,11 +7,11 @@ create table s_00818(a Nullable(Int64), b Nullable(Int64), c Nullable(String)) e insert into t_00818 values(1,1,'a'), (2,2,'b'); insert into s_00818 values(1,1,'a'); -select * from t_00818 left join s_00818 on t_00818.a = s_00818.a; -select * from t_00818 left join s_00818 on t_00818.a = s_00818.a and t_00818.a = s_00818.b; -select * from t_00818 left join s_00818 on t_00818.a = s_00818.a where s_00818.a = 1; -select * from t_00818 left join s_00818 on t_00818.a = s_00818.a and t_00818.a = s_00818.a; -select * from t_00818 left join s_00818 on t_00818.a = s_00818.a and t_00818.b = s_00818.a; +select * from t_00818 left join s_00818 on t_00818.a = s_00818.a ORDER BY t_00818.a; +select * from t_00818 left join s_00818 on t_00818.a = s_00818.a and t_00818.a = s_00818.b ORDER BY t_00818.a; +select * from t_00818 left join s_00818 on t_00818.a = s_00818.a where s_00818.a = 1 ORDER BY t_00818.a; +select * from t_00818 left join s_00818 on t_00818.a = s_00818.a and t_00818.a = s_00818.a ORDER BY t_00818.a; +select * from t_00818 left join s_00818 on t_00818.a = s_00818.a and t_00818.b = s_00818.a ORDER BY t_00818.a; drop table t_00818; drop table s_00818; diff --git a/tests/queries/0_stateless/00820_multiple_joins.sql b/tests/queries/0_stateless/00820_multiple_joins.sql index df82a199337..af13e6f1d8b 100644 --- a/tests/queries/0_stateless/00820_multiple_joins.sql +++ b/tests/queries/0_stateless/00820_multiple_joins.sql @@ -14,69 +14,80 @@ INSERT INTO table2 SELECT number * 2, number * 20 FROM numbers(11); INSERT INTO table3 SELECT number * 30, number * 300 FROM numbers(10); INSERT INTO table5 SELECT number * 5, number * 50, number * 500 FROM numbers(10); -select t1.a, t2.b, t3.c from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on t2.b = t3.b; -select t1.a, t2.b, t5.c from table1 as t1 join table2 as t2 on t1.a = t2.a join table5 as t5 on t1.a = t5.a AND t2.b = t5.b; +select t1.a, t2.b, t3.c from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on t2.b = t3.b ORDER BY t1.a; +select t1.a, t2.b, t5.c from table1 as t1 join table2 as t2 on t1.a = t2.a join table5 as t5 on t1.a = t5.a AND t2.b = t5.b ORDER BY t1.a; select t1.a, t2.a, t2.b, t3.b, t3.c, t5.a, t5.b, t5.c from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on t2.b = t3.b join table5 as t5 on t3.c = t5.c +ORDER BY t1.a FORMAT PrettyCompactNoEscapes; select t1.a as t1_a, t2.a as t2_a, t2.b as t2_b, t3.b as t3_b from table1 as t1 join table2 as t2 on t1_a = t2_a -join table3 as t3 on t2_b = t3_b; +join table3 as t3 on t2_b = t3_b +ORDER BY t1.a +; select t1.a as t1_a, t2.a as t2_a, t2.b as t2_b, t3.b as t3_b from table1 as t1 join table2 as t2 on t1.a = t2.a -join table3 as t3 on t2.b = t3.b; +join table3 as t3 on t2.b = t3.b +ORDER BY t1.a +; select t1.a as t1_a, t2.a as t2_a, t2.b as t2_b, t3.b as t3_b from table1 as t1 join table2 as t2 on table1.a = table2.a -join table3 as t3 on table2.b = table3.b; +join table3 as t3 on table2.b = table3.b +ORDER BY t1.a +; select t1.a, t2.a, t2.b, t3.b from table1 as t1 join table2 as t2 on table1.a = table2.a -join table3 as t3 on table2.b = table3.b; +join table3 as t3 on table2.b = table3.b +ORDER BY t1.a +; select t1.a, t2.a, t2.b, t3.b from table1 as t1 join table2 as t2 on t1.a = t2.a -join table3 as t3 on t2.b = t3.b; +join table3 as t3 on t2.b = t3.b +ORDER BY t1.a +; select table1.a, table2.a, table2.b, table3.b from table1 as t1 join table2 as t2 on table1.a = table2.a -join table3 as t3 on table2.b = table3.b; +join table3 as t3 on table2.b = table3.b +ORDER BY t1.a +; select t1.*, t2.*, t3.* from table1 as t1 join table2 as t2 on table1.a = table2.a join table3 as t3 on table2.b = table3.b +ORDER BY t1.a FORMAT PrettyCompactNoEscapes; select * from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on t2.b = t3.b +ORDER BY t1.a FORMAT PrettyCompactNoEscapes; select t1.a as t1_a, t2.a as t2_a, t2.b as t2_b, t3.b as t3_b, (t1.a + table2.b) as t1_t2_x, (table1.a + table3.b) as t1_t3_x, (t2.b + t3.b) as t2_t3_x from table1 as t1 join table2 as t2 on t1_a = t2_a -join table3 as t3 on t2_b = t3_b; - ---select (t1.a + table2.b) as t1_t2_x, (table1.a + table3.b) as t1_t3_x, (t2.b + t3.b) as t2_t3_x ---from table1 as t1 ---join table2 as t2 on t1_t2_x = t2.a ---join table3 as t3 on t1_t3_x = t2_t3_x; - +join table3 as t3 on t2_b = t3_b +ORDER BY t1.a +; CREATE TABLE table_set ( x UInt32 ) ENGINE = Set; INSERT INTO table_set VALUES (0), (1), (2); diff --git a/tests/queries/0_stateless/00820_multiple_joins_subquery_requires_alias.sql b/tests/queries/0_stateless/00820_multiple_joins_subquery_requires_alias.sql index ad59e02ecad..612e75e6d4d 100644 --- a/tests/queries/0_stateless/00820_multiple_joins_subquery_requires_alias.sql +++ b/tests/queries/0_stateless/00820_multiple_joins_subquery_requires_alias.sql @@ -15,68 +15,73 @@ INSERT INTO table5 SELECT number * 5, number * 50, number * 500 FROM numbers(10) SET joined_subquery_requires_alias = 1; -select t1.a, t2.b, t3.c from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on t2.b = t3.b; -select t1.a, t2.b, t5.c from table1 as t1 join table2 as t2 on t1.a = t2.a join table5 as t5 on t1.a = t5.a AND t2.b = t5.b; +select t1.a, t2.b, t3.c from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on t2.b = t3.b ORDER BY t1.a; +select t1.a, t2.b, t5.c from table1 as t1 join table2 as t2 on t1.a = t2.a join table5 as t5 on t1.a = t5.a AND t2.b = t5.b ORDER BY t1.a; select t1.a, t2.a, t2.b, t3.b, t3.c, t5.a, t5.b, t5.c from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on t2.b = t3.b join table5 as t5 on t3.c = t5.c +ORDER BY t1.a FORMAT PrettyCompactNoEscapes; select t1.a as t1_a, t2.a as t2_a, t2.b as t2_b, t3.b as t3_b from table1 as t1 join table2 as t2 on t1_a = t2_a -join table3 as t3 on t2_b = t3_b; +join table3 as t3 on t2_b = t3_b +ORDER BY t1.a; select t1.a as t1_a, t2.a as t2_a, t2.b as t2_b, t3.b as t3_b from table1 as t1 join table2 as t2 on t1.a = t2.a -join table3 as t3 on t2.b = t3.b; +join table3 as t3 on t2.b = t3.b +ORDER BY t1.a; select t1.a as t1_a, t2.a as t2_a, t2.b as t2_b, t3.b as t3_b from table1 as t1 join table2 as t2 on table1.a = table2.a -join table3 as t3 on table2.b = table3.b; +join table3 as t3 on table2.b = table3.b +ORDER BY t1.a; select t1.a, t2.a, t2.b, t3.b from table1 as t1 join table2 as t2 on table1.a = table2.a -join table3 as t3 on table2.b = table3.b; +join table3 as t3 on table2.b = table3.b +ORDER BY t1.a; select t1.a, t2.a, t2.b, t3.b from table1 as t1 join table2 as t2 on t1.a = t2.a -join table3 as t3 on t2.b = t3.b; +join table3 as t3 on t2.b = t3.b +ORDER BY t1.a; select table1.a, table2.a, table2.b, table3.b from table1 as t1 join table2 as t2 on table1.a = table2.a -join table3 as t3 on table2.b = table3.b; +join table3 as t3 on table2.b = table3.b +ORDER BY t1.a; select t1.*, t2.*, t3.* from table1 as t1 join table2 as t2 on table1.a = table2.a join table3 as t3 on table2.b = table3.b +ORDER BY t1.a FORMAT PrettyCompactNoEscapes; select * from table1 as t1 join table2 as t2 on t1.a = t2.a join table3 as t3 on t2.b = t3.b +ORDER BY t1.a FORMAT PrettyCompactNoEscapes; select t1.a as t1_a, t2.a as t2_a, t2.b as t2_b, t3.b as t3_b, (t1.a + table2.b) as t1_t2_x, (table1.a + table3.b) as t1_t3_x, (t2.b + t3.b) as t2_t3_x from table1 as t1 join table2 as t2 on t1_a = t2_a -join table3 as t3 on t2_b = t3_b; - ---select (t1.a + table2.b) as t1_t2_x, (table1.a + table3.b) as t1_t3_x, (t2.b + t3.b) as t2_t3_x ---from table1 as t1 ---join table2 as t2 on t1_t2_x = t2.a ---join table3 as t3 on t1_t3_x = t2_t3_x; +join table3 as t3 on t2_b = t3_b +ORDER BY t1.a; DROP TABLE table1; DROP TABLE table2; diff --git a/tests/queries/0_stateless/00826_cross_to_inner_join.sql b/tests/queries/0_stateless/00826_cross_to_inner_join.sql index ce0c8ea2bfc..e9f9e13e2d3 100644 --- a/tests/queries/0_stateless/00826_cross_to_inner_join.sql +++ b/tests/queries/0_stateless/00826_cross_to_inner_join.sql @@ -19,9 +19,9 @@ SELECT * FROM t1_00826 cross join t2_00826 where t1_00826.a = t2_00826.a; SELECT '--- cross nullable ---'; SELECT * FROM t1_00826 cross join t2_00826 where t1_00826.b = t2_00826.b; SELECT '--- cross nullable vs not nullable ---'; -SELECT * FROM t1_00826 cross join t2_00826 where t1_00826.a = t2_00826.b; +SELECT * FROM t1_00826 cross join t2_00826 where t1_00826.a = t2_00826.b ORDER BY t1_00826.a; SELECT '--- cross self ---'; -SELECT * FROM t1_00826 x cross join t1_00826 y where x.a = y.a and x.b = y.b; +SELECT * FROM t1_00826 x cross join t1_00826 y where x.a = y.a and x.b = y.b ORDER BY x.a; SELECT '--- cross one table expr ---'; SELECT * FROM t1_00826 cross join t2_00826 where t1_00826.a = t1_00826.b order by (t1_00826.a, t2_00826.a, t2_00826.b); SELECT '--- cross multiple ands ---'; @@ -38,7 +38,7 @@ SELECT '--- arithmetic expr ---'; SELECT * FROM t1_00826 cross join t2_00826 where t1_00826.a + 1 = t2_00826.a + t2_00826.b AND (t1_00826.a + t1_00826.b + t2_00826.a + t2_00826.b > 5); SELECT '--- is null or ---'; -SELECT * FROM t1_00826 cross join t2_00826 where t1_00826.b = t2_00826.a AND (t2_00826.b IS NULL OR t2_00826.b > t2_00826.a); +SELECT * FROM t1_00826 cross join t2_00826 where t1_00826.b = t2_00826.a AND (t2_00826.b IS NULL OR t2_00826.b > t2_00826.a) ORDER BY t1_00826.a; SELECT '--- do not rewrite alias ---'; SELECT a as b FROM t1_00826 cross join t2_00826 where t1_00826.b = t2_00826.a AND b > 0; diff --git a/tests/queries/0_stateless/00952_input_function.sh b/tests/queries/0_stateless/00952_input_function.sh index 54496ba09e0..91aec7ab833 100755 --- a/tests/queries/0_stateless/00952_input_function.sh +++ b/tests/queries/0_stateless/00952_input_function.sh @@ -10,22 +10,22 @@ ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS input_function_table_1" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE input_function_table_1 (a String, b Date, c Int32, d Int16) ENGINE=Memory()" ${CLICKHOUSE_CLIENT} --query="SELECT number, number, number FROM numbers(5) FORMAT CSV" > "${CLICKHOUSE_TMP}"/data_for_input_function.csv cat "${CLICKHOUSE_TMP}"/data_for_input_function.csv | ${CLICKHOUSE_CLIENT} --query="INSERT INTO input_function_table_1 (a, b, c) SELECT a, b, c*c FROM input('a String, b Int32, c Int32') FORMAT CSV" -${CLICKHOUSE_CLIENT} --query="SELECT * FROM input_function_table_1 FORMAT CSV" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM input_function_table_1 FORMAT CSV" | sort ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS input_function_table_2" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE input_function_table_2 (a String, b Date, c Int32, d Int16) ENGINE=Memory()" cat "${CLICKHOUSE_TMP}"/data_for_input_function.csv | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT%20INTO%20input_function_table_2%20%28a%2C%20b%2C%20c%29%20SELECT%20a%2C%20b%2C%20c%2Ac%20FROM%20input%28%27a%20String%2C%20b%20Int32%2C%20c%20Int32%27%29%20FORMAT%20CSV" --data-binary @- -${CLICKHOUSE_CLIENT} --query="SELECT * FROM input_function_table_2 FORMAT CSV" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM input_function_table_2 FORMAT CSV" | sort ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS input_function_table_3" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE input_function_table_3 (a String, b Date, c Int32, d Int16) ENGINE=Memory()" cat "${CLICKHOUSE_TMP}"/data_for_input_function.csv | ${CLICKHOUSE_CLIENT} --query="INSERT INTO input_function_table_3 (a, b, c) SELECT * FROM (SELECT s, b, c*c FROM input('s String, b Int32, c Int32') js1 JOIN input_function_table_1 ON s=input_function_table_1.a) FORMAT CSV" -${CLICKHOUSE_CLIENT} --query="SELECT * FROM input_function_table_3 FORMAT CSV" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM input_function_table_3 FORMAT CSV" | sort ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS input_function_table_4" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE input_function_table_4 (a String, b Date, c Int32, d Int16) ENGINE=Memory()" cat "${CLICKHOUSE_TMP}"/data_for_input_function.csv | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT%20INTO%20input_function_table_4%20%28a%2C%20b%2C%20c%29%20SELECT%20%2A%20FROM%20%28SELECT%20s%2C%20b%2C%20c%2Ac%20FROM%20input%28%27s%20String%2C%20b%20Int32%2C%20c%20Int32%27%29%20js1%20JOIN%20input_function_table_1%20ON%20s%3Dinput_function_table_1.a%29%20FORMAT%20CSV" --data-binary @- -${CLICKHOUSE_CLIENT} --query="SELECT * FROM input_function_table_4 FORMAT CSV" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM input_function_table_4 FORMAT CSV" | sort ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS input_function_table_5" diff --git a/tests/queries/0_stateless/01050_engine_join_crash.sql b/tests/queries/0_stateless/01050_engine_join_crash.sql index 285952056b4..3dd4bd2b798 100644 --- a/tests/queries/0_stateless/01050_engine_join_crash.sql +++ b/tests/queries/0_stateless/01050_engine_join_crash.sql @@ -15,7 +15,7 @@ SELECT * FROM testJoinTable; DROP TABLE testJoinTable; SELECT '-'; - + DROP TABLE IF EXISTS master; DROP TABLE IF EXISTS transaction; @@ -38,8 +38,8 @@ DROP TABLE IF EXISTS tbl; CREATE TABLE tbl (eventDate Date, id String) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY eventDate; CREATE TABLE some_join (id String, value String) ENGINE = Join(ANY, LEFT, id) SETTINGS any_join_distinct_right_table_keys = 1; -SELECT * FROM tbl AS t ANY LEFT JOIN some_join USING (id); -SELECT * FROM tbl AS t ANY LEFT JOIN some_join AS d USING (id); +SELECT * FROM tbl AS t ANY LEFT JOIN some_join USING (id) ORDER BY id; +SELECT * FROM tbl AS t ANY LEFT JOIN some_join AS d USING (id) ORDER BY id; -- TODO SELECT t.*, d.* FROM tbl AS t ANY LEFT JOIN some_join AS d USING (id); DROP TABLE some_join; diff --git a/tests/queries/0_stateless/01051_system_stack_trace.reference b/tests/queries/0_stateless/01051_system_stack_trace.reference index d00491fd7e5..b82bda76142 100644 --- a/tests/queries/0_stateless/01051_system_stack_trace.reference +++ b/tests/queries/0_stateless/01051_system_stack_trace.reference @@ -1 +1,18 @@ +-- { echo } +SELECT count() > 0 FROM system.stack_trace WHERE query_id != ''; +1 +-- opimization for not reading /proc/self/task/{}/comm and avoid sending signal +SELECT countIf(thread_id > 0) > 0 FROM system.stack_trace; +1 +-- optimization for trace +SELECT length(trace) > 0 FROM system.stack_trace LIMIT 1; +1 +-- optimization for query_id +SELECT length(query_id) > 0 FROM system.stack_trace WHERE query_id != '' LIMIT 1; +1 +-- optimization for thread_name +SELECT length(thread_name) > 0 FROM system.stack_trace WHERE thread_name != '' LIMIT 1; +1 +-- enough rows (optimizations works "correctly") +SELECT count() > 100 FROM system.stack_trace; 1 diff --git a/tests/queries/0_stateless/01051_system_stack_trace.sql b/tests/queries/0_stateless/01051_system_stack_trace.sql index e495e2198ea..d018d01fa22 100644 --- a/tests/queries/0_stateless/01051_system_stack_trace.sql +++ b/tests/queries/0_stateless/01051_system_stack_trace.sql @@ -1,4 +1,14 @@ -- Tags: race --- at least this query should be present +-- { echo } SELECT count() > 0 FROM system.stack_trace WHERE query_id != ''; +-- opimization for not reading /proc/self/task/{}/comm and avoid sending signal +SELECT countIf(thread_id > 0) > 0 FROM system.stack_trace; +-- optimization for trace +SELECT length(trace) > 0 FROM system.stack_trace LIMIT 1; +-- optimization for query_id +SELECT length(query_id) > 0 FROM system.stack_trace WHERE query_id != '' LIMIT 1; +-- optimization for thread_name +SELECT length(thread_name) > 0 FROM system.stack_trace WHERE thread_name != '' LIMIT 1; +-- enough rows (optimizations works "correctly") +SELECT count() > 100 FROM system.stack_trace; diff --git a/tests/queries/0_stateless/01576_alias_column_rewrite.sql b/tests/queries/0_stateless/01576_alias_column_rewrite.sql index 450127797cc..8424eb11f9b 100644 --- a/tests/queries/0_stateless/01576_alias_column_rewrite.sql +++ b/tests/queries/0_stateless/01576_alias_column_rewrite.sql @@ -25,8 +25,6 @@ SELECT t = '2020-01-03' FROM (SELECT day AS t FROM test_table WHERE t = '2020-01 SELECT COUNT() = 10 FROM test_table WHERE day = '2020-01-01' UNION ALL SELECT 1 FROM numbers(1) SETTINGS max_rows_to_read = 11; SELECT COUNT() = 0 FROM (SELECT toDate('2019-01-01') AS day, day AS t FROM test_table PREWHERE t = '2020-01-03' WHERE t = '2020-01-03' GROUP BY t ); - - SELECT 'test-join'; SELECT day = '2020-01-03' FROM diff --git a/tests/queries/0_stateless/01655_plan_optimizations.sh b/tests/queries/0_stateless/01655_plan_optimizations.sh index 3a3ce95460c..aaecdc390cb 100755 --- a/tests/queries/0_stateless/01655_plan_optimizations.sh +++ b/tests/queries/0_stateless/01655_plan_optimizations.sh @@ -172,7 +172,7 @@ $CLICKHOUSE_CLIENT -q " $CLICKHOUSE_CLIENT -q " select number as a, r.b from numbers(4) as l any left join ( select number + 2 as b from numbers(3) - ) as r on a = r.b where a != 1 and b != 2 settings enable_optimize_predicate_expression = 0" + ) as r on a = r.b where a != 1 and b != 2 settings enable_optimize_predicate_expression = 0" | sort echo "> one condition of filter is pushed down before INNER JOIN" $CLICKHOUSE_CLIENT -q " diff --git a/tests/queries/0_stateless/01910_view_dictionary.sql b/tests/queries/0_stateless/01910_view_dictionary.sql index 7b807c7fbba..1f9928735b4 100644 --- a/tests/queries/0_stateless/01910_view_dictionary.sql +++ b/tests/queries/0_stateless/01910_view_dictionary.sql @@ -21,9 +21,11 @@ CREATE TABLE dictionary_source_ru INSERT INTO dictionary_source_ru VALUES (1, 'Один'), (2,'Два'), (3, 'Три'); -CREATE VIEW dictionary_source_view AS SELECT id, dictionary_source_en.value as value_en, dictionary_source_ru.value as value_ru FROM dictionary_source_en LEFT JOIN dictionary_source_ru USING (id); +CREATE VIEW dictionary_source_view AS + SELECT id, dictionary_source_en.value as value_en, dictionary_source_ru.value as value_ru + FROM dictionary_source_en LEFT JOIN dictionary_source_ru USING (id); -select * from dictionary_source_view; +select * from dictionary_source_view ORDER BY id; CREATE DICTIONARY flat_dictionary ( diff --git a/tests/queries/0_stateless/01913_join_push_down_bug.sql b/tests/queries/0_stateless/01913_join_push_down_bug.sql index 00fd8cb5ddb..b945530b32a 100644 --- a/tests/queries/0_stateless/01913_join_push_down_bug.sql +++ b/tests/queries/0_stateless/01913_join_push_down_bug.sql @@ -17,6 +17,7 @@ set query_plan_filter_push_down = true; SELECT id, flag FROM test t1 INNER JOIN (SELECT DISTINCT id FROM test) AS t2 ON t1.id = t2.id -WHERE flag = 0 and t = 1 AND id NOT IN (SELECT 1 WHERE 0); +WHERE flag = 0 and t = 1 AND id NOT IN (SELECT 1 WHERE 0) +ORDER BY id; DROP TABLE IF EXISTS test; diff --git a/tests/queries/0_stateless/02115_rewrite_local_join_right_distribute_table.reference b/tests/queries/0_stateless/02115_rewrite_local_join_right_distribute_table.reference index b9119b9d087..2b974451e74 100644 --- a/tests/queries/0_stateless/02115_rewrite_local_join_right_distribute_table.reference +++ b/tests/queries/0_stateless/02115_rewrite_local_join_right_distribute_table.reference @@ -8,11 +8,12 @@ SELECT a FROM t1_all AS t1 ALL INNER JOIN test_02115.t2_local AS t2 ON a = t2.a 1 -2 -3 1 2 +2 3 +3 +- 1 2 3 @@ -24,13 +25,13 @@ FROM t1_all AS t1 GLOBAL ALL INNER JOIN t2_all AS t2 ON a = t2.a 1 1 -2 -2 -3 -3 1 1 2 2 +2 +2 +3 +3 3 3 diff --git a/tests/queries/0_stateless/02115_rewrite_local_join_right_distribute_table.sql b/tests/queries/0_stateless/02115_rewrite_local_join_right_distribute_table.sql index 2eebb14a46f..d5ab82ba064 100644 --- a/tests/queries/0_stateless/02115_rewrite_local_join_right_distribute_table.sql +++ b/tests/queries/0_stateless/02115_rewrite_local_join_right_distribute_table.sql @@ -13,18 +13,20 @@ create table t2_local as t1_local; create table t1_all as t1_local engine Distributed(test_cluster_two_shards_localhost, test_02115, t1_local, rand()); create table t2_all as t2_local engine Distributed(test_cluster_two_shards_localhost, test_02115, t2_local, rand()); -insert into t1_local values(1), (2), (3); -insert into t2_local values(1), (2), (3); +insert into t1_local values (1), (2), (3); +insert into t2_local values (1), (2), (3); set distributed_product_mode = 'local'; select * from t1_all t1 where t1.a in (select t2.a from t2_all t2); explain syntax select t1.* from t1_all t1 join t2_all t2 on t1.a = t2.a; -select t1.* from t1_all t1 join t2_all t2 on t1.a = t2.a; +select t1.* from t1_all t1 join t2_all t2 on t1.a = t2.a ORDER BY t1.a; + +SELECT '-'; set distributed_product_mode = 'global'; select * from t1_all t1 where t1.a in (select t2.a from t2_all t2); explain syntax select t1.* from t1_all t1 join t2_all t2 on t1.a = t2.a; -select t1.* from t1_all t1 join t2_all t2 on t1.a = t2.a; +select t1.* from t1_all t1 join t2_all t2 on t1.a = t2.a ORDER BY t1.a; DROP TABLE t1_local; DROP TABLE t2_local; diff --git a/tests/queries/0_stateless/02297_regex_parsing_file_names.reference b/tests/queries/0_stateless/02297_regex_parsing_file_names.reference new file mode 100644 index 00000000000..b4de3947675 --- /dev/null +++ b/tests/queries/0_stateless/02297_regex_parsing_file_names.reference @@ -0,0 +1 @@ +11 diff --git a/tests/queries/0_stateless/02297_regex_parsing_file_names.sh b/tests/queries/0_stateless/02297_regex_parsing_file_names.sh new file mode 100755 index 00000000000..12ccb54235b --- /dev/null +++ b/tests/queries/0_stateless/02297_regex_parsing_file_names.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Tags: no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Data preparation. + +# Now we can get the user_files_path by use the table file function for trick. also we can get it by query as: +# "insert into function file('exist.txt', 'CSV', 'val1 char') values ('aaaa'); select _path from file('exist.txt', 'CSV', 'val1 char')" +CLICKHOUSE_USER_FILES_PATH=$(clickhouse-client --query "select _path, _file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +mkdir -p ${CLICKHOUSE_USER_FILES_PATH}/ + +rm -rf ${CLICKHOUSE_USER_FILES_PATH}/file_{0..10}.csv + +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_0.csv +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_1.csv +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_2.csv +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_3.csv +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_4.csv +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_5.csv +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_6.csv +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_7.csv +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_8.csv +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_9.csv +echo '0' > ${CLICKHOUSE_USER_FILES_PATH}/file_10.csv + +# echo '' > ${CLICKHOUSE_USER_FILES_PATH}/file_10.csv + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_regex;" + +${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_regex (id UInt64) ENGINE = MergeTree() order by id;" + +${CLICKHOUSE_CLIENT} -q "INSERT INTO t_regex SELECT * FROM file('file_{0..10}.csv','CSV');" +${CLICKHOUSE_CLIENT} -q "SELECT count() from t_regex;" + +rm -rf ${CLICKHOUSE_USER_FILES_PATH}/file_{0..10}.csv; +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_regex;" diff --git a/tests/queries/0_stateless/02353_translate.reference b/tests/queries/0_stateless/02353_translate.reference new file mode 100644 index 00000000000..557b5182127 --- /dev/null +++ b/tests/queries/0_stateless/02353_translate.reference @@ -0,0 +1,16 @@ +Hello, world! +cagaacgttc +jihgfe +jihgff +jihgfg +jihgfh +jihgfi +HotelGenev +ードとは +¿йðՅন𐐏 +¿йðՅনন +¿йðՅনՅ +¿йðՅনð +¿йðՅনй +abc +abc diff --git a/tests/queries/0_stateless/02353_translate.sql b/tests/queries/0_stateless/02353_translate.sql new file mode 100644 index 00000000000..a7059ec85a7 --- /dev/null +++ b/tests/queries/0_stateless/02353_translate.sql @@ -0,0 +1,13 @@ +SELECT translate('Hello? world.', '.?', '!,'); +SELECT translate('gtcttgcaag', 'ACGTacgt', 'TGCAtgca'); +SELECT translate(toString(number), '0123456789', 'abcdefghij') FROM numbers(987654, 5); + +SELECT translateUTF8('HôtelGenèv', 'Ááéíóúôè', 'aaeiouoe'); +SELECT translateUTF8('中文内码', '久标准中文内码', 'ユニコードとは'); +SELECT translateUTF8(toString(number), '1234567890', 'ዩय𐑿𐐏নՅðй¿ค') FROM numbers(987654, 5); + +SELECT translate('abc', '', ''); +SELECT translateUTF8('abc', '', ''); + +SELECT translate('abc', 'Ááéíóúôè', 'aaeiouoe'); -- { serverError 36 } +SELECT translateUTF8('abc', 'efg', ''); -- { serverError 36 } diff --git a/tests/queries/0_stateless/02354_window_expression_with_aggregation_expression.reference b/tests/queries/0_stateless/02354_window_expression_with_aggregation_expression.reference new file mode 100644 index 00000000000..494a1a0cd49 --- /dev/null +++ b/tests/queries/0_stateless/02354_window_expression_with_aggregation_expression.reference @@ -0,0 +1,15 @@ +100 +100 +-- { echoOn } +SELECT arrayMap(x -> (x + 1), groupArray(number) OVER ()) AS result +FROM numbers(10); +[1,2,3,4,5,6,7,8,9,10] +[1,2,3,4,5,6,7,8,9,10] +[1,2,3,4,5,6,7,8,9,10] +[1,2,3,4,5,6,7,8,9,10] +[1,2,3,4,5,6,7,8,9,10] +[1,2,3,4,5,6,7,8,9,10] +[1,2,3,4,5,6,7,8,9,10] +[1,2,3,4,5,6,7,8,9,10] +[1,2,3,4,5,6,7,8,9,10] +[1,2,3,4,5,6,7,8,9,10] diff --git a/tests/queries/0_stateless/02354_window_expression_with_aggregation_expression.sql b/tests/queries/0_stateless/02354_window_expression_with_aggregation_expression.sql new file mode 100644 index 00000000000..21da3c5f403 --- /dev/null +++ b/tests/queries/0_stateless/02354_window_expression_with_aggregation_expression.sql @@ -0,0 +1,16 @@ +SELECT + sum(a)*100/sum(sum(a)) OVER (PARTITION BY b) AS r +FROM +( + SELECT 1 AS a, 2 AS b + UNION ALL + SELECT 3 AS a, 4 AS b + UNION ALL + SELECT 5 AS a, 2 AS b + +) AS t +GROUP BY b; + +-- { echoOn } +SELECT arrayMap(x -> (x + 1), groupArray(number) OVER ()) AS result +FROM numbers(10); diff --git a/tests/queries/0_stateless/02356_trivial_count_with_empty_set.reference b/tests/queries/0_stateless/02356_trivial_count_with_empty_set.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02356_trivial_count_with_empty_set.sql b/tests/queries/0_stateless/02356_trivial_count_with_empty_set.sql new file mode 100644 index 00000000000..89630d1aa4a --- /dev/null +++ b/tests/queries/0_stateless/02356_trivial_count_with_empty_set.sql @@ -0,0 +1,9 @@ +drop table if exists test; + +create table test(a Int64) Engine=MergeTree order by tuple(); + +set optimize_trivial_count_query=1, empty_result_for_aggregation_by_empty_set=1; + +select count() from test; + +drop table test; diff --git a/tests/queries/0_stateless/02357_query_cancellation_race.reference b/tests/queries/0_stateless/02357_query_cancellation_race.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02357_query_cancellation_race.sh b/tests/queries/0_stateless/02357_query_cancellation_race.sh new file mode 100755 index 00000000000..6b20e050ce3 --- /dev/null +++ b/tests/queries/0_stateless/02357_query_cancellation_race.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Tags: race + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "create table tab (x UInt64, y String) engine = MergeTree order by x" +for _ in $(seq 1 100); do timeout -s 2 0.05 $CLICKHOUSE_CLIENT --interactive_delay 1000 -q "insert into tab select number, toString(number) from system.numbers" || true; done diff --git a/tests/queries/0_stateless/02359_send_logs_source_regexp.reference b/tests/queries/0_stateless/02359_send_logs_source_regexp.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02359_send_logs_source_regexp.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02359_send_logs_source_regexp.sh b/tests/queries/0_stateless/02359_send_logs_source_regexp.sh new file mode 100755 index 00000000000..d3b60bc59f4 --- /dev/null +++ b/tests/queries/0_stateless/02359_send_logs_source_regexp.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +[ ! -z "$CLICKHOUSE_CLIENT_REDEFINED" ] && CLICKHOUSE_CLIENT=$CLICKHOUSE_CLIENT_REDEFINED + +CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=trace/g') +regexp="executeQuery|InterpreterSelectQuery" +$CLICKHOUSE_CLIENT --send_logs_source_regexp "$regexp" -q "SELECT 1;" 2> >(grep -v -E "$regexp" 1>&2) diff --git a/tests/queries/0_stateless/02360_send_logs_level_colors.reference b/tests/queries/0_stateless/02360_send_logs_level_colors.reference new file mode 100644 index 00000000000..fe2824243c4 --- /dev/null +++ b/tests/queries/0_stateless/02360_send_logs_level_colors.reference @@ -0,0 +1,3 @@ +ASCII text +ASCII text +ASCII text diff --git a/tests/queries/0_stateless/02360_send_logs_level_colors.sh b/tests/queries/0_stateless/02360_send_logs_level_colors.sh new file mode 100755 index 00000000000..4e5ce057702 --- /dev/null +++ b/tests/queries/0_stateless/02360_send_logs_level_colors.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +file_name="${CLICKHOUSE_TMP}/res_${CLICKHOUSE_DATABASE}.log" +CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=trace/g') + +# Run query via expect to make isatty() return true +function run() +{ + command=$1 + expect << EOF +log_user 0 +set timeout 3 +match_max 100000 + +spawn bash -c "$command" +expect 1 +EOF + + file "$file_name" | grep -o "ASCII text" + file "$file_name" | grep -o "with escape sequences" +} + +run "$CLICKHOUSE_CLIENT -q 'SELECT 1' 2>$file_name" +run "$CLICKHOUSE_CLIENT -q 'SELECT 1' --server_logs_file=$file_name" +run "$CLICKHOUSE_CLIENT -q 'SELECT 1' --server_logs_file=- >$file_name" + +rm -f "$file_name" diff --git a/tests/queries/0_stateless/02362_part_log_merge_algorithm.reference b/tests/queries/0_stateless/02362_part_log_merge_algorithm.reference new file mode 100644 index 00000000000..91a959d4255 --- /dev/null +++ b/tests/queries/0_stateless/02362_part_log_merge_algorithm.reference @@ -0,0 +1,5 @@ +data_horizontal all_1_1_0 NewPart Undecided +data_horizontal all_1_1_1 MergeParts Horizontal +data_vertical all_1_1_0 NewPart Undecided +data_vertical all_2_2_0 NewPart Undecided +data_vertical all_1_2_1 MergeParts Vertical diff --git a/tests/queries/0_stateless/02362_part_log_merge_algorithm.sql b/tests/queries/0_stateless/02362_part_log_merge_algorithm.sql new file mode 100644 index 00000000000..6446b46c393 --- /dev/null +++ b/tests/queries/0_stateless/02362_part_log_merge_algorithm.sql @@ -0,0 +1,26 @@ +CREATE TABLE data_horizontal ( + key Int +) +Engine=MergeTree() +ORDER BY key; + +INSERT INTO data_horizontal VALUES (1); +OPTIMIZE TABLE data_horizontal FINAL; +SYSTEM FLUSH LOGS; +SELECT table, part_name, event_type, merge_algorithm FROM system.part_log WHERE event_date >= yesterday() AND database = currentDatabase() AND table = 'data_horizontal' ORDER BY event_time_microseconds; + +CREATE TABLE data_vertical +( + key UInt64, + value String +) +ENGINE = MergeTree() +ORDER BY key +SETTINGS index_granularity_bytes = 0, enable_mixed_granularity_parts = 0, min_bytes_for_wide_part = 0, +vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 1; + +INSERT INTO data_vertical VALUES (1, '1'); +INSERT INTO data_vertical VALUES (2, '2'); +OPTIMIZE TABLE data_vertical FINAL; +SYSTEM FLUSH LOGS; +SELECT table, part_name, event_type, merge_algorithm FROM system.part_log WHERE event_date >= yesterday() AND database = currentDatabase() AND table = 'data_vertical' ORDER BY event_time_microseconds; diff --git a/tests/queries/0_stateless/02363_mapupdate_improve.reference b/tests/queries/0_stateless/02363_mapupdate_improve.reference new file mode 100644 index 00000000000..04e2b943929 --- /dev/null +++ b/tests/queries/0_stateless/02363_mapupdate_improve.reference @@ -0,0 +1,10 @@ +{'fruit':'apple','season':'autumn'} +{'fruit':'apple','season':'autumn'} +{'fruit':'apple','season':'autumn'} +{'fruit':'apple','season':'autumn'} +{'fruit':'apple','season':'autumn'} +{'season':'autumn','fruit':'apple'} +{'season':'autumn','fruit':'apple'} +{'season':'autumn','fruit':'apple'} +{'season':'autumn','fruit':'apple'} +{'season':'autumn','fruit':'apple'} diff --git a/tests/queries/0_stateless/02363_mapupdate_improve.sql b/tests/queries/0_stateless/02363_mapupdate_improve.sql new file mode 100644 index 00000000000..6b7723cc9b4 --- /dev/null +++ b/tests/queries/0_stateless/02363_mapupdate_improve.sql @@ -0,0 +1,11 @@ +-- Tags: no-backward-compatibility-check +DROP TABLE IF EXISTS map_test; +CREATE TABLE map_test(`tags` Map(String, String)) ENGINE = MergeTree PRIMARY KEY tags ORDER BY tags SETTINGS index_granularity = 8192; +INSERT INTO map_test (tags) VALUES (map('fruit','apple','color','red')); +INSERT INTO map_test (tags) VALUES (map('fruit','apple','color','red')); +INSERT INTO map_test (tags) VALUES (map('fruit','apple','color','red')); +INSERT INTO map_test (tags) VALUES (map('fruit','apple','color','red')); +INSERT INTO map_test (tags) VALUES (map('fruit','apple','color','red')); +SELECT mapUpdate(mapFilter((k, v) -> (k in ('fruit')), tags), map('season', 'autumn')) FROM map_test; +SELECT mapUpdate(map('season','autumn'), mapFilter((k, v) -> (k in ('fruit')), tags)) FROM map_test; +DROP TABLE map_test;