mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge branch 'master' into cmake-self-extracting-executable
This commit is contained in:
Yakov Olkhovskiy
GitHub
commit
e7438cc433
@ -8,7 +8,7 @@ concurrency:
|
||||
group: cherry-pick
|
||||
on: # yamllint disable-line rule:truthy
|
||||
schedule:
|
||||
- cron: '0 */3 * * *'
|
||||
- cron: '0 * * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
16
.github/workflows/master.yml
vendored
16
.github/workflows/master.yml
vendored
@ -2902,7 +2902,7 @@ jobs:
|
||||
#############################################################################################
|
||||
#################################### PERFORMANCE TESTS ######################################
|
||||
#############################################################################################
|
||||
PerformanceComparison0:
|
||||
PerformanceComparisonX86-0:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -2940,7 +2940,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison1:
|
||||
PerformanceComparisonX86-1:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -2978,7 +2978,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison2:
|
||||
PerformanceComparisonX86-2:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3016,7 +3016,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison3:
|
||||
PerformanceComparisonX86-3:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3096,10 +3096,10 @@ jobs:
|
||||
- IntegrationTestsTsan1
|
||||
- IntegrationTestsTsan2
|
||||
- IntegrationTestsTsan3
|
||||
- PerformanceComparison0
|
||||
- PerformanceComparison1
|
||||
- PerformanceComparison2
|
||||
- PerformanceComparison3
|
||||
- PerformanceComparisonX86-0
|
||||
- PerformanceComparisonX86-1
|
||||
- PerformanceComparisonX86-2
|
||||
- PerformanceComparisonX86-3
|
||||
- CompatibilityCheck
|
||||
- ASTFuzzerTestDebug
|
||||
- ASTFuzzerTestAsan
|
||||
|
||||
32
.github/workflows/pull_request.yml
vendored
32
.github/workflows/pull_request.yml
vendored
@ -3118,7 +3118,7 @@ jobs:
|
||||
#############################################################################################
|
||||
#################################### PERFORMANCE TESTS ######################################
|
||||
#############################################################################################
|
||||
PerformanceComparison0:
|
||||
PerformanceComparisonX86-0:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3156,7 +3156,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison1:
|
||||
PerformanceComparisonX86-1:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3194,7 +3194,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison2:
|
||||
PerformanceComparisonX86-2:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3232,7 +3232,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparison3:
|
||||
PerformanceComparisonX86-3:
|
||||
needs: [BuilderDebRelease]
|
||||
runs-on: [self-hosted, stress-tester]
|
||||
steps:
|
||||
@ -3270,7 +3270,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparisonAarch0:
|
||||
PerformanceComparisonAarch-0:
|
||||
needs: [BuilderDebAarch64]
|
||||
runs-on: [self-hosted, func-tester-aarch64]
|
||||
steps:
|
||||
@ -3308,7 +3308,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparisonAarch1:
|
||||
PerformanceComparisonAarch-1:
|
||||
needs: [BuilderDebAarch64]
|
||||
runs-on: [self-hosted, func-tester-aarch64]
|
||||
steps:
|
||||
@ -3346,7 +3346,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparisonAarch2:
|
||||
PerformanceComparisonAarch-2:
|
||||
needs: [BuilderDebAarch64]
|
||||
runs-on: [self-hosted, func-tester-aarch64]
|
||||
steps:
|
||||
@ -3384,7 +3384,7 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
PerformanceComparisonAarch3:
|
||||
PerformanceComparisonAarch-3:
|
||||
needs: [BuilderDebAarch64]
|
||||
runs-on: [self-hosted, func-tester-aarch64]
|
||||
steps:
|
||||
@ -3481,14 +3481,14 @@ jobs:
|
||||
- IntegrationTestsTsan1
|
||||
- IntegrationTestsTsan2
|
||||
- IntegrationTestsTsan3
|
||||
- PerformanceComparison0
|
||||
- PerformanceComparison1
|
||||
- PerformanceComparison2
|
||||
- PerformanceComparison3
|
||||
- PerformanceComparisonAarch0
|
||||
- PerformanceComparisonAarch1
|
||||
- PerformanceComparisonAarch2
|
||||
- PerformanceComparisonAarch3
|
||||
- PerformanceComparisonX86-0
|
||||
- PerformanceComparisonX86-1
|
||||
- PerformanceComparisonX86-2
|
||||
- PerformanceComparisonX86-3
|
||||
- PerformanceComparisonAarch-0
|
||||
- PerformanceComparisonAarch-1
|
||||
- PerformanceComparisonAarch-2
|
||||
- PerformanceComparisonAarch-3
|
||||
- UnitTestsAsan
|
||||
- UnitTestsTsan
|
||||
- UnitTestsMsan
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
cmake_minimum_required(VERSION 3.14)
|
||||
cmake_minimum_required(VERSION 3.15)
|
||||
|
||||
project(ClickHouse LANGUAGES C CXX ASM)
|
||||
|
||||
@ -84,18 +84,13 @@ message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
|
||||
string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
|
||||
|
||||
option(USE_STATIC_LIBRARIES "Disable to use shared libraries" ON)
|
||||
# DEVELOPER ONLY.
|
||||
# Faster linking if turned on.
|
||||
option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files" OFF)
|
||||
option(CLICKHOUSE_SPLIT_BINARY "Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled" OFF)
|
||||
|
||||
if (NOT USE_STATIC_LIBRARIES)
|
||||
# DEVELOPER ONLY.
|
||||
# Faster linking if turned on.
|
||||
option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files")
|
||||
|
||||
option(CLICKHOUSE_SPLIT_BINARY
|
||||
"Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled")
|
||||
endif ()
|
||||
|
||||
if (USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
|
||||
message(FATAL_ERROR "Defining SPLIT_SHARED_LIBRARIES=1 without USE_STATIC_LIBRARIES=0 has no effect.")
|
||||
if (USE_STATIC_LIBRARIES AND (SPLIT_SHARED_LIBRARIES OR CLICKHOUSE_SPLIT_BINARY))
|
||||
message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 or CLICKHOUSE_SPLIT_BINARY=1 must not be used together with USE_STATIC_LIBRARIES=1")
|
||||
endif()
|
||||
|
||||
if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
|
||||
@ -345,6 +340,22 @@ if (COMPILER_GCC OR COMPILER_CLANG)
|
||||
set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
|
||||
endif ()
|
||||
|
||||
if (ARCH_AMD64)
|
||||
# align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
|
||||
# which makes benchmark results more stable.
|
||||
set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
|
||||
if (COMPILER_GCC)
|
||||
# gcc is in assembler, need to add "-Wa," prefix
|
||||
set(BRANCHES_WITHIN_32B_BOUNDARIES "-Wa,${BRANCHES_WITHIN_32B_BOUNDARIES}")
|
||||
endif()
|
||||
|
||||
include(CheckCXXCompilerFlag)
|
||||
check_cxx_compiler_flag("${BRANCHES_WITHIN_32B_BOUNDARIES}" HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
|
||||
if (HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
|
||||
set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (COMPILER_GCC)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines")
|
||||
endif ()
|
||||
|
||||
@ -12,7 +12,14 @@
|
||||
#define JSON_MAX_DEPTH 100
|
||||
|
||||
|
||||
#ifdef __clang__
|
||||
# pragma clang diagnostic push
|
||||
# pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec"
|
||||
#endif
|
||||
POCO_IMPLEMENT_EXCEPTION(JSONException, Poco::Exception, "JSONException") // NOLINT(cert-err60-cpp, modernize-use-noexcept, hicpp-use-noexcept)
|
||||
#ifdef __clang__
|
||||
# pragma clang diagnostic pop
|
||||
#endif
|
||||
|
||||
|
||||
/// Прочитать беззнаковое целое в простом формате из не-0-terminated строки.
|
||||
|
||||
@ -38,8 +38,14 @@
|
||||
*/
|
||||
|
||||
|
||||
#ifdef __clang__
|
||||
# pragma clang diagnostic push
|
||||
# pragma clang diagnostic ignored "-Wdeprecated-dynamic-exception-spec"
|
||||
#endif
|
||||
POCO_DECLARE_EXCEPTION(Foundation_API, JSONException, Poco::Exception)
|
||||
|
||||
#ifdef __clang__
|
||||
# pragma clang diagnostic pop
|
||||
#endif
|
||||
|
||||
class JSON
|
||||
{
|
||||
|
||||
@ -93,7 +93,6 @@
|
||||
# define NO_SANITIZE_ADDRESS __attribute__((__no_sanitize__("address")))
|
||||
# define NO_SANITIZE_THREAD __attribute__((__no_sanitize__("thread")))
|
||||
# define ALWAYS_INLINE_NO_SANITIZE_UNDEFINED __attribute__((__always_inline__, __no_sanitize__("undefined")))
|
||||
# define DISABLE_SANITIZER_INSTRUMENTATION __attribute__((disable_sanitizer_instrumentation))
|
||||
#else /// It does not work in GCC. GCC 7 cannot recognize this attribute and GCC 8 simply ignores it.
|
||||
# define NO_SANITIZE_UNDEFINED
|
||||
# define NO_SANITIZE_ADDRESS
|
||||
@ -101,6 +100,13 @@
|
||||
# define ALWAYS_INLINE_NO_SANITIZE_UNDEFINED ALWAYS_INLINE
|
||||
#endif
|
||||
|
||||
#if defined(__clang__) && defined(__clang_major__) && __clang_major__ >= 14
|
||||
# define DISABLE_SANITIZER_INSTRUMENTATION __attribute__((disable_sanitizer_instrumentation))
|
||||
#else
|
||||
# define DISABLE_SANITIZER_INSTRUMENTATION
|
||||
#endif
|
||||
|
||||
|
||||
#if !__has_include(<sanitizer/asan_interface.h>) || !defined(ADDRESS_SANITIZER)
|
||||
# define ASAN_UNPOISON_MEMORY_REGION(a, b)
|
||||
# define ASAN_POISON_MEMORY_REGION(a, b)
|
||||
|
||||
@ -19,6 +19,7 @@ option (ENABLE_POPCNT "Use popcnt instructions on x86_64" 1)
|
||||
option (ENABLE_AVX "Use AVX instructions on x86_64" 0)
|
||||
option (ENABLE_AVX2 "Use AVX2 instructions on x86_64" 0)
|
||||
option (ENABLE_AVX512 "Use AVX512 instructions on x86_64" 0)
|
||||
option (ENABLE_AVX512_VBMI "Use AVX512_VBMI instruction on x86_64 (depends on ENABLE_AVX512)" 0)
|
||||
option (ENABLE_BMI "Use BMI instructions on x86_64" 0)
|
||||
option (ENABLE_AVX2_FOR_SPEC_OP "Use avx2 instructions for specific operations on x86_64" 0)
|
||||
option (ENABLE_AVX512_FOR_SPEC_OP "Use avx512 instructions for specific operations on x86_64" 0)
|
||||
@ -151,6 +152,20 @@ elseif (ARCH_AMD64)
|
||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
|
||||
endif ()
|
||||
|
||||
set (TEST_FLAG "-mavx512vbmi")
|
||||
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
|
||||
check_cxx_source_compiles("
|
||||
#include <immintrin.h>
|
||||
int main() {
|
||||
auto a = _mm512_permutexvar_epi8(__m512i(), __m512i());
|
||||
(void)a;
|
||||
return 0;
|
||||
}
|
||||
" HAVE_AVX512_VBMI)
|
||||
if (HAVE_AVX512 AND ENABLE_AVX512 AND HAVE_AVX512_VBMI AND ENABLE_AVX512_VBMI)
|
||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
|
||||
endif ()
|
||||
|
||||
set (TEST_FLAG "-mbmi")
|
||||
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
|
||||
check_cxx_source_compiles("
|
||||
|
||||
@ -1,3 +1,19 @@
|
||||
if (_CLICKHOUSE_TOOLCHAIN_FILE_LOADED)
|
||||
# During first run of cmake the toolchain file will be loaded twice,
|
||||
# - /usr/share/cmake-3.23/Modules/CMakeDetermineSystem.cmake
|
||||
# - /bld/CMakeFiles/3.23.2/CMakeSystem.cmake
|
||||
#
|
||||
# But once you already have non-empty cmake cache it will be loaded only
|
||||
# once:
|
||||
# - /bld/CMakeFiles/3.23.2/CMakeSystem.cmake
|
||||
#
|
||||
# This has no harm except for double load of toolchain will add
|
||||
# --gcc-toolchain multiple times that will not allow ccache to reuse the
|
||||
# cache.
|
||||
return()
|
||||
endif()
|
||||
set (_CLICKHOUSE_TOOLCHAIN_FILE_LOADED ON)
|
||||
|
||||
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
|
||||
|
||||
set (CMAKE_SYSTEM_NAME "Linux")
|
||||
|
||||
@ -20,16 +20,11 @@ if (COMPILER_CLANG)
|
||||
# We want to get everything out of the compiler for code quality.
|
||||
add_warning(everything)
|
||||
add_warning(pedantic)
|
||||
no_warning(vla-extension)
|
||||
no_warning(zero-length-array)
|
||||
no_warning(c11-extensions)
|
||||
no_warning(unused-command-line-argument)
|
||||
no_warning(c++98-compat-pedantic)
|
||||
no_warning(c++98-compat)
|
||||
no_warning(c99-extensions)
|
||||
no_warning(conversion)
|
||||
no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
|
||||
no_warning(deprecated-dynamic-exception-spec)
|
||||
no_warning(disabled-macro-expansion)
|
||||
no_warning(documentation-unknown-command)
|
||||
no_warning(double-promotion)
|
||||
@ -38,12 +33,7 @@ if (COMPILER_CLANG)
|
||||
no_warning(global-constructors)
|
||||
no_warning(missing-prototypes)
|
||||
no_warning(missing-variable-declarations)
|
||||
no_warning(nested-anon-types)
|
||||
no_warning(packed)
|
||||
no_warning(padded)
|
||||
no_warning(return-std-move-in-c++11) # clang 7+
|
||||
no_warning(shift-sign-overflow)
|
||||
no_warning(sign-conversion)
|
||||
no_warning(switch-enum)
|
||||
no_warning(undefined-func-template)
|
||||
no_warning(unused-template)
|
||||
|
||||
@ -2,7 +2,7 @@ set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest")
|
||||
|
||||
add_library(_gtest "${SRC_DIR}/src/gtest-all.cc")
|
||||
set_target_properties(_gtest PROPERTIES VERSION "1.0.0")
|
||||
target_compile_definitions (_gtest INTERFACE GTEST_HAS_POSIX_RE=0)
|
||||
target_compile_definitions (_gtest PUBLIC GTEST_HAS_POSIX_RE=0)
|
||||
target_include_directories(_gtest SYSTEM PUBLIC "${SRC_DIR}/include")
|
||||
target_include_directories(_gtest PRIVATE "${SRC_DIR}")
|
||||
|
||||
|
||||
@ -93,6 +93,18 @@ set (CMAKE_CXX_STANDARD 17)
|
||||
set (LLVM_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm")
|
||||
set (LLVM_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm")
|
||||
add_subdirectory ("${LLVM_SOURCE_DIR}" "${LLVM_BINARY_DIR}")
|
||||
set_directory_properties (PROPERTIES
|
||||
# due to llvm crosscompile cmake does not know how to clean it, and on clean
|
||||
# will lead to the following error:
|
||||
#
|
||||
# ninja: error: remove(contrib/llvm/llvm/NATIVE): Directory not empty
|
||||
#
|
||||
ADDITIONAL_CLEAN_FILES "${LLVM_BINARY_DIR}"
|
||||
# llvm's cmake configuring this file only when cmake runs,
|
||||
# and after clean cmake will not know that it should re-run,
|
||||
# add explicitly depends from llvm-config.h
|
||||
CMAKE_CONFIGURE_DEPENDS "${LLVM_BINARY_DIR}/include/llvm/Config/llvm-config.h"
|
||||
)
|
||||
|
||||
add_library (_llvm INTERFACE)
|
||||
target_link_libraries (_llvm INTERFACE ${REQUIRED_LLVM_LIBRARIES})
|
||||
|
||||
2
contrib/thrift
vendored
2
contrib/thrift
vendored
@ -1 +1 @@
|
||||
Subproject commit 010ccf0a0c7023fea0f6bf4e4078ebdff7e61982
|
||||
Subproject commit 2a93df80f27739ccabb5b885cb12a8dc7595ecdf
|
||||
@ -15,7 +15,6 @@ set(thriftcpp_SOURCES
|
||||
"${LIBRARY_DIR}/src/thrift/async/TConcurrentClientSyncInfo.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/concurrency/ThreadManager.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/concurrency/TimerManager.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/concurrency/Util.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/processor/PeekProcessor.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/protocol/TBase64Utils.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/protocol/TDebugProtocol.cpp"
|
||||
@ -33,6 +32,8 @@ set(thriftcpp_SOURCES
|
||||
"${LIBRARY_DIR}/src/thrift/transport/TServerSocket.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/transport/TTransportUtils.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/transport/TBufferTransports.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/transport/SocketCommon.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/transport/TWebSocketServer.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/server/TConnectedClient.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/server/TServerFramework.cpp"
|
||||
"${LIBRARY_DIR}/src/thrift/server/TSimpleServer.cpp"
|
||||
@ -92,4 +93,4 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}")
|
||||
add_library(_thrift ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
|
||||
add_library(ch_contrib::thrift ALIAS _thrift)
|
||||
target_include_directories(_thrift SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src" ${CMAKE_CURRENT_BINARY_DIR})
|
||||
target_link_libraries (_thrift PUBLIC boost::headers_only)
|
||||
target_link_libraries (_thrift PUBLIC OpenSSL::SSL boost::headers_only)
|
||||
|
||||
@ -29,17 +29,21 @@ env
|
||||
|
||||
if [ -n "$MAKE_DEB" ]; then
|
||||
rm -rf /build/packages/root
|
||||
if [ -z "$SANITIZER" ]; then
|
||||
# We need to check if clickhouse-diagnostics is fine and build it
|
||||
(
|
||||
cd /build/programs/diagnostics
|
||||
make test-no-docker
|
||||
GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build
|
||||
mv clickhouse-diagnostics ..
|
||||
)
|
||||
else
|
||||
echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics
|
||||
chmod +x /build/programs/clickhouse-diagnostics
|
||||
# NOTE: this is for backward compatibility with previous releases,
|
||||
# that does not diagnostics tool (only script).
|
||||
if [ -d /build/programs/diagnostics ]; then
|
||||
if [ -z "$SANITIZER" ]; then
|
||||
# We need to check if clickhouse-diagnostics is fine and build it
|
||||
(
|
||||
cd /build/programs/diagnostics
|
||||
make test-no-docker
|
||||
GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build
|
||||
mv clickhouse-diagnostics ..
|
||||
)
|
||||
else
|
||||
echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics
|
||||
chmod +x /build/programs/clickhouse-diagnostics
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@ -71,6 +71,7 @@ RUN apt-get update \
|
||||
software-properties-common \
|
||||
tzdata \
|
||||
unixodbc \
|
||||
file \
|
||||
--yes --no-install-recommends
|
||||
|
||||
RUN pip3 install numpy scipy pandas Jinja2
|
||||
|
||||
@ -34,6 +34,7 @@ RUN apt-get update -y \
|
||||
unixodbc \
|
||||
wget \
|
||||
zstd \
|
||||
file \
|
||||
&& apt-get clean
|
||||
|
||||
|
||||
|
||||
@ -322,7 +322,11 @@ else
|
||||
clickhouse-client --query="SELECT 'Server version: ', version()"
|
||||
|
||||
# Install new package before running stress test because we should use new clickhouse-client and new clickhouse-test
|
||||
# But we should leave old binary in /usr/bin/ for gdb (so it will print sane stacktarces)
|
||||
mv /usr/bin/clickhouse previous_release_package_folder/
|
||||
install_packages package_folder
|
||||
mv /usr/bin/clickhouse package_folder/
|
||||
mv previous_release_package_folder/clickhouse /usr/bin/
|
||||
|
||||
mkdir tmp_stress_output
|
||||
|
||||
@ -337,6 +341,7 @@ else
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
|
||||
|
||||
# Start new server
|
||||
mv package_folder/clickhouse /usr/bin/
|
||||
configure
|
||||
start 500
|
||||
clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
|
||||
@ -354,6 +359,11 @@ else
|
||||
# Error messages (we should ignore some errors)
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 ("Cannot parse string 'Hello' as UInt64")
|
||||
# FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
|
||||
# Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
|
||||
# NOTE Incompatibility was introduced in https://github.com/ClickHouse/ClickHouse/pull/39263, it's expected
|
||||
# ("This engine is deprecated and is not supported in transactions", "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part")
|
||||
echo "Check for Error messages in server log:"
|
||||
zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
|
||||
-e "Code: 236. DB::Exception: Cancelled mutating parts" \
|
||||
@ -378,6 +388,11 @@ else
|
||||
-e "is lost forever." \
|
||||
-e "Unknown index: idx." \
|
||||
-e "Cannot parse string 'Hello' as UInt64" \
|
||||
-e "} <Error> TCPHandler: Code:" \
|
||||
-e "} <Error> executeQuery: Code:" \
|
||||
-e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
|
||||
-e "This engine is deprecated and is not supported in transactions" \
|
||||
-e "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part" \
|
||||
/var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
|
||||
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
|
||||
|
||||
@ -46,6 +46,9 @@ def get_options(i, backward_compatibility_check):
|
||||
if i == 13:
|
||||
client_options.append("memory_tracker_fault_probability=0.001")
|
||||
|
||||
if i % 2 == 1 and not backward_compatibility_check:
|
||||
client_options.append("group_by_use_nulls=1")
|
||||
|
||||
if client_options:
|
||||
options.append(" --client-option " + " ".join(client_options))
|
||||
|
||||
|
||||
@ -7,31 +7,31 @@ DIR=
|
||||
|
||||
if [ "${OS}" = "Linux" ]
|
||||
then
|
||||
if [ "${ARCH}" = "x86_64" ]
|
||||
if [ "${ARCH}" = "x86_64" -o "${ARCH}" = "amd64" ]
|
||||
then
|
||||
DIR="amd64"
|
||||
elif [ "${ARCH}" = "aarch64" ]
|
||||
elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ]
|
||||
then
|
||||
DIR="aarch64"
|
||||
elif [ "${ARCH}" = "powerpc64le" ] || [ "${ARCH}" = "ppc64le" ]
|
||||
elif [ "${ARCH}" = "powerpc64le" -o "${ARCH}" = "ppc64le" ]
|
||||
then
|
||||
DIR="powerpc64le"
|
||||
fi
|
||||
elif [ "${OS}" = "FreeBSD" ]
|
||||
then
|
||||
if [ "${ARCH}" = "x86_64" ]
|
||||
if [ "${ARCH}" = "x86_64" -o "${ARCH}" = "amd64" ]
|
||||
then
|
||||
DIR="freebsd"
|
||||
elif [ "${ARCH}" = "aarch64" ]
|
||||
elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ]
|
||||
then
|
||||
DIR="freebsd-aarch64"
|
||||
elif [ "${ARCH}" = "powerpc64le" ] || [ "${ARCH}" = "ppc64le" ]
|
||||
elif [ "${ARCH}" = "powerpc64le" -o "${ARCH}" = "ppc64le" ]
|
||||
then
|
||||
DIR="freebsd-powerpc64le"
|
||||
fi
|
||||
elif [ "${OS}" = "Darwin" ]
|
||||
then
|
||||
if [ "${ARCH}" = "x86_64" ]
|
||||
if [ "${ARCH}" = "x86_64" -o "${ARCH}" = "amd64" ]
|
||||
then
|
||||
DIR="macos"
|
||||
elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ]
|
||||
|
||||
@ -75,7 +75,7 @@ This will create the `programs/clickhouse` executable, which can be used with `c
|
||||
The build requires the following components:
|
||||
|
||||
- Git (is used only to checkout the sources, it’s not needed for the build)
|
||||
- CMake 3.14 or newer
|
||||
- CMake 3.15 or newer
|
||||
- Ninja
|
||||
- C++ compiler: clang-14 or newer
|
||||
- Linker: lld
|
||||
|
||||
@ -1632,6 +1632,8 @@ kafka_topic_list = 'topic1',
|
||||
kafka_group_name = 'group1',
|
||||
kafka_format = 'AvroConfluent';
|
||||
|
||||
-- for debug purposes you can set format_avro_schema_registry_url in a session.
|
||||
-- this way cannot be used in production
|
||||
SET format_avro_schema_registry_url = 'http://schema-registry';
|
||||
|
||||
SELECT * FROM topic1_stream;
|
||||
|
||||
@ -2626,7 +2626,7 @@ Possible values:
|
||||
- Any positive integer.
|
||||
- 0 - Disabled (infinite timeout).
|
||||
|
||||
Default value: 1800.
|
||||
Default value: 180.
|
||||
|
||||
## http_receive_timeout {#http_receive_timeout}
|
||||
|
||||
@ -2637,7 +2637,7 @@ Possible values:
|
||||
- Any positive integer.
|
||||
- 0 - Disabled (infinite timeout).
|
||||
|
||||
Default value: 1800.
|
||||
Default value: 180.
|
||||
|
||||
## check_query_single_value_result {#check_query_single_value_result}
|
||||
|
||||
@ -3329,6 +3329,15 @@ Read more about [memory overcommit](memory-overcommit.md).
|
||||
|
||||
Default value: `1GiB`.
|
||||
|
||||
## compatibility {#compatibility}
|
||||
|
||||
This setting changes other settings according to provided ClickHouse version.
|
||||
If a behaviour in ClickHouse was changed by using a different default value for some setting, this compatibility setting allows you to use default values from previous versions for all the settings that were not set by the user.
|
||||
|
||||
This setting takes ClickHouse version number as a string, like `21.3`, `21.8`. Empty value means that this setting is disabled.
|
||||
|
||||
Disabled by default.
|
||||
|
||||
# Format settings {#format-settings}
|
||||
|
||||
## input_format_skip_unknown_fields {#input_format_skip_unknown_fields}
|
||||
|
||||
@ -19,6 +19,10 @@ The `system.part_log` table contains the following columns:
|
||||
- `REGULAR_MERGE` — Some regular merge.
|
||||
- `TTL_DELETE_MERGE` — Cleaning up expired data.
|
||||
- `TTL_RECOMPRESS_MERGE` — Recompressing data part with the.
|
||||
- `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Merge algorithm for the event with type `MERGE_PARTS`. Can have one of the following values:
|
||||
- `UNDECIDED`
|
||||
- `HORIZONTAL`
|
||||
- `VERTICAL`
|
||||
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
|
||||
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
|
||||
- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision.
|
||||
@ -52,6 +56,7 @@ Row 1:
|
||||
query_id: 983ad9c7-28d5-4ae1-844e-603116b7de31
|
||||
event_type: NewPart
|
||||
merge_reason: NotAMerge
|
||||
merge_algorithm: Undecided
|
||||
event_date: 2021-02-02
|
||||
event_time: 2021-02-02 11:14:28
|
||||
event_time_microseconds: 2021-02-02 11:14:28.861919
|
||||
|
||||
@ -1715,3 +1715,7 @@ Return value type is always [Float64](../../sql-reference/data-types/float.md).
|
||||
│ 6 │ Float64 │
|
||||
└─────┴──────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Distance functions
|
||||
|
||||
All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).
|
||||
|
||||
502
docs/en/sql-reference/functions/distance-functions.md
Normal file
502
docs/en/sql-reference/functions/distance-functions.md
Normal file
@ -0,0 +1,502 @@
|
||||
# Distance functions
|
||||
|
||||
## L1Norm
|
||||
|
||||
Calculates the sum of absolute values of a vector.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Norm(vector)
|
||||
```
|
||||
|
||||
Alias: `normL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance.
|
||||
|
||||
Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Norm((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Norm((1, 2))─┐
|
||||
│ 3 │
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
## L2Norm
|
||||
|
||||
Calculates the square root of the sum of the squares of the vector values.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Norm(vector)
|
||||
```
|
||||
|
||||
Alias: `normL2`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance).
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Norm((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌───L2Norm((1, 2))─┐
|
||||
│ 2.23606797749979 │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
## LinfNorm
|
||||
|
||||
Calculates the maximum of absolute values of a vector.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfNorm(vector)
|
||||
```
|
||||
|
||||
Alias: `normLinf`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Linf-norm or the maximum absolute value.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfNorm((1, -2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfNorm((1, -2))─┐
|
||||
│ 2 │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
## LpNorm
|
||||
|
||||
Calculates the root of `p`-th power of the sum of the absolute values of a vector in the power of `p`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpNorm(vector, p)
|
||||
```
|
||||
|
||||
Alias: `normLp`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `p` — The power. Possible values: real number in `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpNorm((1, -2), 2);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpNorm((1, -2), 2)─┐
|
||||
│ 2.23606797749979 │
|
||||
└────────────────────┘
|
||||
```
|
||||
|
||||
## L1Distance
|
||||
|
||||
Calculates the distance between two points (the values of the vectors are the coordinates) in `L1` space (1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Distance(vector1, vector2)
|
||||
```
|
||||
|
||||
Alias: `distanceL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- 1-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Distance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Distance((1, 2), (2, 3))─┐
|
||||
│ 2 │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## L2Distance
|
||||
|
||||
Calculates the distance between two points (the values of the vectors are the coordinates) in Euclidean space ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Distance(vector1, vector2)
|
||||
```
|
||||
|
||||
Alias: `distanceL2`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- 2-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Distance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L2Distance((1, 2), (2, 3))─┐
|
||||
│ 1.4142135623730951 │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## LinfDistance
|
||||
|
||||
Calculates the distance between two points (the values of the vectors are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfDistance(vector1, vector2)
|
||||
```
|
||||
|
||||
Alias: `distanceLinf`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `vector1` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Infinity-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfDistance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfDistance((1, 2), (2, 3))─┐
|
||||
│ 1 │
|
||||
└──────────────────────────────┘
|
||||
```
|
||||
|
||||
## LpDistance
|
||||
|
||||
Calculates the distance between two points (the values of the vectors are the coordinates) in `Lp` space ([p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpDistance(vector1, vector2, p)
|
||||
```
|
||||
|
||||
Alias: `distanceLp`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `vector1` — First vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `vector2` — Second vector. [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
- `p` — The power. Possible values: real number from `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- p-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpDistance((1, 2), (2, 3), 3);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpDistance((1, 2), (2, 3), 3)─┐
|
||||
│ 1.2599210498948732 │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
## L1Normalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L1` space ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Normalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Normalize((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Normalize((1, 2))─────────────────────┐
|
||||
│ (0.3333333333333333,0.6666666666666666) │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## L2Normalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in Euclidean space (using [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Normalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Normalize((3, 4));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L2Normalize((3, 4))─┐
|
||||
│ (0.6,0.8) │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## LinfNormalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L_{inf}` space (using [maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfNormalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeLinf `.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfNormalize((3, 4));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfNormalize((3, 4))─┐
|
||||
│ (0.75,1) │
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
## LpNormalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `Lp` space (using [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpNormalize(tuple, p)
|
||||
```
|
||||
|
||||
Alias: `normalizeLp `.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpNormalize((3, 4),5);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpNormalize((3, 4), 5)──────────────────┐
|
||||
│ (0.7187302630182624,0.9583070173576831) │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## cosineDistance
|
||||
|
||||
Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
cosineDistance(tuple1, tuple2)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Cosine of the angle between two vectors substracted from one.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT cosineDistance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─cosineDistance((1, 2), (2, 3))─┐
|
||||
│ 0.007722123286332261 │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
@ -681,6 +681,47 @@ SELECT
|
||||
└────────────┴─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## parseTimeDelta
|
||||
|
||||
Parse a sequence of numbers followed by something resembling a time unit.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
parseTimeDelta(timestr)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `timestr` — A sequence of numbers followed by something resembling a time unit.
|
||||
|
||||
|
||||
**Returned value**
|
||||
|
||||
- A floating-point number with the number of seconds.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT parseTimeDelta('11s+22min')
|
||||
```
|
||||
|
||||
```text
|
||||
┌─parseTimeDelta('11s+22min')─┐
|
||||
│ 1331 │
|
||||
└─────────────────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT parseTimeDelta('1yr2mo')
|
||||
```
|
||||
|
||||
```text
|
||||
┌─parseTimeDelta('1yr2mo')─┐
|
||||
│ 36806400 │
|
||||
└──────────────────────────┘
|
||||
```
|
||||
|
||||
## least(a, b)
|
||||
|
||||
Returns the smallest value from a and b.
|
||||
|
||||
@ -93,3 +93,35 @@ Predefined characters: `\0`, `\\`, `|`, `(`, `)`, `^`, `$`, `.`, `[`, `]`, `?`,
|
||||
This implementation slightly differs from re2::RE2::QuoteMeta. It escapes zero byte as `\0` instead of `\x00` and it escapes only required characters.
|
||||
For more information, see the link: [RE2](https://github.com/google/re2/blob/master/re2/re2.cc#L473)
|
||||
|
||||
|
||||
## translate(s, from, to)
|
||||
|
||||
The function replaces characters in the string ‘s’ in accordance with one-to-one character mapping defined by ‘from’ and ‘to’ strings. ‘from’ and ‘to’ must be ASCII strings of the same size. Non-ASCII characters in the original string are not modified.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT translate('Hello, World!', 'delor', 'DELOR') AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ HELLO, WORLD! │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
## translateUTF8(string, from, to)
|
||||
|
||||
Similar to previous function, but works with UTF-8 arguments. ‘from’ and ‘to’ must be valid UTF-8 strings of the same size.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT translateUTF8('Hélló, Wórld¡', 'óé¡', 'oe!') AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ Hello, World! │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
@ -374,7 +374,7 @@ The same as `multiMatchAny`, but returns any index that matches the haystack.
|
||||
|
||||
## multiMatchAllIndices(haystack, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
|
||||
|
||||
The same as `multiMatchAny`, but returns the array of all indicies that match the haystack in any order.
|
||||
The same as `multiMatchAny`, but returns the array of all indices that match the haystack in any order.
|
||||
|
||||
## multiFuzzyMatchAny(haystack, distance, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
|
||||
|
||||
|
||||
@ -559,502 +559,7 @@ Result:
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## L1Norm
|
||||
|
||||
Calculates the sum of absolute values of a tuple.
|
||||
## Distance functions
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Norm(tuple)
|
||||
```
|
||||
|
||||
Alias: `normL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance.
|
||||
|
||||
Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Norm((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Norm((1, 2))─┐
|
||||
│ 3 │
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
## L2Norm
|
||||
|
||||
Calculates the square root of the sum of the squares of the tuple values.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Norm(tuple)
|
||||
```
|
||||
|
||||
Alias: `normL2`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance).
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Norm((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌───L2Norm((1, 2))─┐
|
||||
│ 2.23606797749979 │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
## LinfNorm
|
||||
|
||||
Calculates the maximum of absolute values of a tuple.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfNorm(tuple)
|
||||
```
|
||||
|
||||
Alias: `normLinf`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Linf-norm or the maximum absolute value.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfNorm((1, -2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfNorm((1, -2))─┐
|
||||
│ 2 │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
## LpNorm
|
||||
|
||||
Calculates the root of `p`-th power of the sum of the absolute values of a tuple in the power of `p`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpNorm(tuple, p)
|
||||
```
|
||||
|
||||
Alias: `normLp`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `p` — The power. Possible values: real number in `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpNorm((1, -2), 2);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpNorm((1, -2), 2)─┐
|
||||
│ 2.23606797749979 │
|
||||
└────────────────────┘
|
||||
```
|
||||
|
||||
## L1Distance
|
||||
|
||||
Calculates the distance between two points (the values of the tuples are the coordinates) in `L1` space (1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Distance(tuple1, tuple2)
|
||||
```
|
||||
|
||||
Alias: `distanceL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- 1-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Distance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Distance((1, 2), (2, 3))─┐
|
||||
│ 2 │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## L2Distance
|
||||
|
||||
Calculates the distance between two points (the values of the tuples are the coordinates) in Euclidean space ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Distance(tuple1, tuple2)
|
||||
```
|
||||
|
||||
Alias: `distanceL2`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- 2-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Distance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L2Distance((1, 2), (2, 3))─┐
|
||||
│ 1.4142135623730951 │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## LinfDistance
|
||||
|
||||
Calculates the distance between two points (the values of the tuples are the coordinates) in `L_{inf}` space ([maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfDistance(tuple1, tuple2)
|
||||
```
|
||||
|
||||
Alias: `distanceLinf`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Infinity-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfDistance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfDistance((1, 2), (2, 3))─┐
|
||||
│ 1 │
|
||||
└──────────────────────────────┘
|
||||
```
|
||||
|
||||
## LpDistance
|
||||
|
||||
Calculates the distance between two points (the values of the tuples are the coordinates) in `Lp` space ([p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpDistance(tuple1, tuple2, p)
|
||||
```
|
||||
|
||||
Alias: `distanceLp`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `p` — The power. Possible values: real number from `[1; inf)`. [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- p-norm distance.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpDistance((1, 2), (2, 3), 3);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpDistance((1, 2), (2, 3), 3)─┐
|
||||
│ 1.2599210498948732 │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
## L1Normalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L1` space ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L1Normalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L1Normalize((1, 2));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L1Normalize((1, 2))─────────────────────┐
|
||||
│ (0.3333333333333333,0.6666666666666666) │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## L2Normalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in Euclidean space (using [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
L2Normalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeL1`.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT L2Normalize((3, 4));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─L2Normalize((3, 4))─┐
|
||||
│ (0.6,0.8) │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## LinfNormalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `L_{inf}` space (using [maximum norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm))).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LinfNormalize(tuple)
|
||||
```
|
||||
|
||||
Alias: `normalizeLinf `.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LinfNormalize((3, 4));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LinfNormalize((3, 4))─┐
|
||||
│ (0.75,1) │
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
## LpNormalize
|
||||
|
||||
Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in `Lp` space (using [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
LpNormalize(tuple, p)
|
||||
```
|
||||
|
||||
Alias: `normalizeLp `.
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Unit vector.
|
||||
|
||||
Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT LpNormalize((3, 4),5);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─LpNormalize((3, 4), 5)──────────────────┐
|
||||
│ (0.7187302630182624,0.9583070173576831) │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## cosineDistance
|
||||
|
||||
Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
cosineDistance(tuple1, tuple2)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Cosine of the angle between two vectors substracted from one.
|
||||
|
||||
Type: [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT cosineDistance((1, 2), (2, 3));
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─cosineDistance((1, 2), (2, 3))─┐
|
||||
│ 0.007722123286332261 │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).
|
||||
|
||||
@ -108,7 +108,8 @@ Note that data will be deleted neither from `table1` nor from `table2`.
|
||||
For the query to run successfully, the following conditions must be met:
|
||||
|
||||
- Both tables must have the same structure.
|
||||
- Both tables must have the same partition key.
|
||||
- Both tables must have the same partition key, the same order by key and the same primary key.
|
||||
- Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables).
|
||||
|
||||
## REPLACE PARTITION
|
||||
|
||||
@ -121,7 +122,8 @@ This query copies the data partition from the `table1` to `table2` and replaces
|
||||
For the query to run successfully, the following conditions must be met:
|
||||
|
||||
- Both tables must have the same structure.
|
||||
- Both tables must have the same partition key.
|
||||
- Both tables must have the same partition key, the same order by key and the same primary key.
|
||||
- Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables).
|
||||
|
||||
## MOVE PARTITION TO TABLE
|
||||
|
||||
@ -134,9 +136,9 @@ This query moves the data partition from the `table_source` to `table_dest` with
|
||||
For the query to run successfully, the following conditions must be met:
|
||||
|
||||
- Both tables must have the same structure.
|
||||
- Both tables must have the same partition key.
|
||||
- Both tables must have the same partition key, the same order by key and the same primary key.
|
||||
- Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables).
|
||||
- Both tables must be the same engine family (replicated or non-replicated).
|
||||
- Both tables must have the same storage policy.
|
||||
|
||||
## CLEAR COLUMN IN PARTITION
|
||||
|
||||
|
||||
@ -5,7 +5,7 @@ sidebar_label: null function
|
||||
|
||||
# null
|
||||
|
||||
Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately droped right after the query execution. The function is used for the convenience of test writing and demonstrations.
|
||||
Creates a temporary table of the specified structure with the [Null](../../engines/table-engines/special/null.md) table engine. According to the `Null`-engine properties, the table data is ignored and the table itself is immediately dropped right after the query execution. The function is used for the convenience of test writing and demonstrations.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -40,4 +40,3 @@ See also:
|
||||
|
||||
- [Null table engine](../../engines/table-engines/special/null.md)
|
||||
|
||||
[Original article](https://clickhouse.com/docs/en/sql-reference/table-functions/null/) <!--hide-->
|
||||
|
||||
@ -91,50 +91,50 @@ SELECT * FROM remote_table;
|
||||
This example uses one table from a sample dataset. The database is `imdb`, and the table is `actors`.
|
||||
|
||||
#### On the source ClickHouse system (the system that currently hosts the data)
|
||||
- Verify the source database and table name (`imdb.actors`)
|
||||
```sql
|
||||
show databases
|
||||
```
|
||||
- Verify the source database and table name (`imdb.actors`)
|
||||
```sql
|
||||
show databases
|
||||
```
|
||||
|
||||
```sql
|
||||
show tables in imdb
|
||||
```
|
||||
```sql
|
||||
show tables in imdb
|
||||
```
|
||||
|
||||
- Get the CREATE TABLE statement from the source:
|
||||
```
|
||||
select create_table_query
|
||||
from system.tables
|
||||
where database = 'imdb' and table = 'actors'
|
||||
```
|
||||
```
|
||||
select create_table_query
|
||||
from system.tables
|
||||
where database = 'imdb' and table = 'actors'
|
||||
```
|
||||
|
||||
Response
|
||||
```sql
|
||||
CREATE TABLE imdb.actors (`id` UInt32,
|
||||
`first_name` String,
|
||||
`last_name` String,
|
||||
`gender` FixedString(1))
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
|
||||
ORDER BY (id, first_name, last_name, gender)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
Response
|
||||
```sql
|
||||
CREATE TABLE imdb.actors (`id` UInt32,
|
||||
`first_name` String,
|
||||
`last_name` String,
|
||||
`gender` FixedString(1))
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
|
||||
ORDER BY (id, first_name, last_name, gender)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
#### On the destination ClickHouse system:
|
||||
|
||||
- Create the destination database:
|
||||
```sql
|
||||
CREATE DATABASE imdb
|
||||
```
|
||||
```sql
|
||||
CREATE DATABASE imdb
|
||||
```
|
||||
|
||||
- Using the CREATE TABLE statement from the source, create the destination:
|
||||
```sql
|
||||
CREATE TABLE imdb.actors (`id` UInt32,
|
||||
`first_name` String,
|
||||
`last_name` String,
|
||||
`gender` FixedString(1))
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
|
||||
ORDER BY (id, first_name, last_name, gender)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
```sql
|
||||
CREATE TABLE imdb.actors (`id` UInt32,
|
||||
`first_name` String,
|
||||
`last_name` String,
|
||||
`gender` FixedString(1))
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
|
||||
ORDER BY (id, first_name, last_name, gender)
|
||||
SETTINGS index_granularity = 8192
|
||||
```
|
||||
|
||||
#### Back on the source deployment:
|
||||
|
||||
@ -145,7 +145,7 @@ remoteSecure('remote.clickhouse.cloud:9440', 'imdb.actors', 'USER', 'PASSWORD',
|
||||
SELECT * from imdb.actors
|
||||
```
|
||||
|
||||
## Globs in Addresses {globs-in-addresses}
|
||||
## Globs in Addresses {#globs-in-addresses}
|
||||
|
||||
Patterns in curly brackets `{ }` are used to generate a set of shards and to specify replicas. If there are multiple pairs of curly brackets, then the direct product of the corresponding sets is generated.
|
||||
The following pattern types are supported.
|
||||
|
||||
@ -335,7 +335,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234
|
||||
|
||||
Поддерживаемые типы данных: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`.
|
||||
|
||||
Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem).
|
||||
Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions#hasany), [hasAll](../../../sql-reference/functions/array-functions#hasall).
|
||||
|
||||
**Примеры**
|
||||
|
||||
|
||||
@ -14,6 +14,15 @@
|
||||
- `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition).
|
||||
- `MUTATE_PART` — изменение куска.
|
||||
- `MOVE_PART` — перемещение куска между дисками.
|
||||
- `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — Причина события с типом `MERGE_PARTS`. Может принимать одно из следующих значений:
|
||||
- `NOT_A_MERGE` — событие имеет тип иной, чем `MERGE_PARTS`.
|
||||
- `REGULAR_MERGE` — обычное слияние.
|
||||
- `TTL_DELETE_MERGE` — очистка истекших данных.
|
||||
- `TTL_RECOMPRESS_MERGE` — переупаковка куска.
|
||||
- `merge_algorithm` ([Enum8](../../sql-reference/data-types/enum.md)) — Алгоритм слияния для события с типом `MERGE_PARTS`. Может принимать одно из следующих значений:
|
||||
- `UNDECIDED`
|
||||
- `HORIZONTAL`
|
||||
- `VERTICAL`
|
||||
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата события.
|
||||
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время события.
|
||||
- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время события с точностью до микросекунд.
|
||||
@ -46,6 +55,8 @@ Row 1:
|
||||
──────
|
||||
query_id: 983ad9c7-28d5-4ae1-844e-603116b7de31
|
||||
event_type: NewPart
|
||||
merge_reason: NotAMerge
|
||||
merge_algorithm: Undecided
|
||||
event_date: 2021-02-02
|
||||
event_time: 2021-02-02 11:14:28
|
||||
event_time_microseconds: 2021-02-02 11:14:28.861919
|
||||
|
||||
@ -83,3 +83,34 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## translate(s, from, to)
|
||||
|
||||
Данная функция заменяет символы в строке ‘s’ в соответствии с поэлементным отображением определяемым строками ‘from’ и ‘to’. ‘from’ и ‘to’ должны быть корректными ASCII строками одного размера. Не ASCII символы в оригинальной строке не изменяются.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT translate('Hello, World!', 'delor', 'DELOR') AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ HELLO, WORLD! │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
## translateUTF8(string, from, to)
|
||||
|
||||
Аналогично предыдущей функции, но работает со строками, состоящими из UTF-8 символов. ‘from’ и ‘to’ должны быть корректными UTF-8 строками одного размера.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT translateUTF8('Hélló, Wórld¡', 'óé¡', 'oe!') AS res
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─res───────────┐
|
||||
│ Hello, World! │
|
||||
└───────────────┘
|
||||
```
|
||||
|
||||
@ -106,7 +106,8 @@ ALTER TABLE table2 [ON CLUSTER cluster] ATTACH PARTITION partition_expr FROM tab
|
||||
Следует иметь в виду:
|
||||
|
||||
- Таблицы должны иметь одинаковую структуру.
|
||||
- Для таблиц должен быть задан одинаковый ключ партиционирования.
|
||||
- Для таблиц должен быть задан одинаковый ключ партиционирования, одинаковый ключ сортировки и одинаковый первичный ключ.
|
||||
- Для таблиц должна быть задана одинаковая политика хранения (диск, на котором хранится партиция, должен быть доступен для обеих таблиц).
|
||||
|
||||
Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr).
|
||||
|
||||
@ -121,7 +122,8 @@ ALTER TABLE table2 [ON CLUSTER cluster] REPLACE PARTITION partition_expr FROM ta
|
||||
Следует иметь в виду:
|
||||
|
||||
- Таблицы должны иметь одинаковую структуру.
|
||||
- Для таблиц должен быть задан одинаковый ключ партиционирования.
|
||||
- Для таблиц должен быть задан одинаковый ключ партиционирования, одинаковый ключ сортировки и одинаковый первичный ключ.
|
||||
- Для таблиц должна быть задана одинаковая политика хранения (диск, на котором хранится партиция, должен быть доступен для обеих таблиц).
|
||||
|
||||
Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr).
|
||||
|
||||
@ -136,9 +138,9 @@ ALTER TABLE table_source [ON CLUSTER cluster] MOVE PARTITION partition_expr TO T
|
||||
Следует иметь в виду:
|
||||
|
||||
- Таблицы должны иметь одинаковую структуру.
|
||||
- Для таблиц должен быть задан одинаковый ключ партиционирования.
|
||||
- Для таблиц должен быть задан одинаковый ключ партиционирования, одинаковый ключ сортировки и одинаковый первичный ключ.
|
||||
- Для таблиц должна быть задана одинаковая политика хранения (диск, на котором хранится партиция, должен быть доступен для обеих таблиц).
|
||||
- Движки таблиц должны быть одинакового семейства (реплицированные или нереплицированные).
|
||||
- Для таблиц должна быть задана одинаковая политика хранения.
|
||||
|
||||
## CLEAR COLUMN IN PARTITION {#alter_clear-column-partition}
|
||||
|
||||
|
||||
@ -106,7 +106,9 @@ void Client::processError(const String & query) const
|
||||
std::vector<String> Client::loadWarningMessages()
|
||||
{
|
||||
std::vector<String> messages;
|
||||
connection->sendQuery(connection_parameters.timeouts, "SELECT message FROM system.warnings", "" /* query_id */,
|
||||
connection->sendQuery(connection_parameters.timeouts,
|
||||
"SELECT * FROM viewIfPermitted(SELECT message FROM system.warnings ELSE null('message String'))",
|
||||
"" /* query_id */,
|
||||
QueryProcessingStage::Complete,
|
||||
&global_context->getSettingsRef(),
|
||||
&global_context->getClientInfo(), false, {});
|
||||
|
||||
@ -20,7 +20,7 @@ public:
|
||||
|
||||
bool operator<=(const WrappingUInt32 & other) const
|
||||
{
|
||||
const UInt32 HALF = 1 << 31;
|
||||
const UInt32 HALF = static_cast<UInt32>(1) << 31;
|
||||
return (value <= other.value && other.value - value < HALF)
|
||||
|| (value > other.value && value - other.value > HALF);
|
||||
}
|
||||
|
||||
@ -154,7 +154,7 @@ int DisksApp::main(const std::vector<String> & /*args*/)
|
||||
Poco::Logger::root().setLevel(Poco::Logger::parseLevel(log_level));
|
||||
|
||||
auto log_path = config().getString("logger.clickhouse-disks", "/var/log/clickhouse-server/clickhouse-disks.log");
|
||||
Poco::Logger::root().setChannel(new Poco::FileChannel(log_path));
|
||||
Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::FileChannel>(new Poco::FileChannel(log_path)));
|
||||
}
|
||||
|
||||
if (config().has("config-file") || fs::exists(getDefaultConfigFileName()))
|
||||
|
||||
@ -604,9 +604,23 @@
|
||||
if this setting is true the user B will see all rows, and if this setting is false the user B will see no rows.
|
||||
By default this setting is false for compatibility with earlier access configurations. -->
|
||||
<users_without_row_policies_can_read_rows>false</users_without_row_policies_can_read_rows>
|
||||
|
||||
<!-- By default, for backward compatibility ON CLUSTER queries ignore CLUSTER grant,
|
||||
however you can change this behaviour by setting this to true -->
|
||||
<on_cluster_queries_require_cluster_grant>false</on_cluster_queries_require_cluster_grant>
|
||||
|
||||
<!-- By default, for backward compatibility "SELECT * FROM system.<table>" doesn't require any grants and can be executed
|
||||
by any user. You can change this behaviour by setting this to true.
|
||||
If it's set to true then this query requires "GRANT SELECT ON system.<table>" just like as for non-system tables.
|
||||
Exceptions: a few system tables ("tables", "columns", "databases", and some constant tables like "one", "contributors")
|
||||
are still accessible for everyone; and if there is a SHOW privilege (e.g. "SHOW USERS") granted the corresponding system
|
||||
table (i.e. "system.users") will be accessible. -->
|
||||
<select_from_system_db_requires_grant>false</select_from_system_db_requires_grant>
|
||||
|
||||
<!-- By default, for backward compatibility "SELECT * FROM information_schema.<table>" doesn't require any grants and can be
|
||||
executed by any user. You can change this behaviour by setting this to true.
|
||||
If it's set to true then this query requires "GRANT SELECT ON information_schema.<table>" just like as for ordinary tables. -->
|
||||
<select_from_information_schema_requires_grant>false</select_from_information_schema_requires_grant>
|
||||
</access_control_improvements>
|
||||
|
||||
<!-- Default profile of settings. -->
|
||||
|
||||
@ -101,7 +101,7 @@ public:
|
||||
registered_prefixes = prefixes_;
|
||||
}
|
||||
|
||||
bool isSettingNameAllowed(const std::string_view & setting_name) const
|
||||
bool isSettingNameAllowed(std::string_view setting_name) const
|
||||
{
|
||||
if (Settings::hasBuiltin(setting_name))
|
||||
return true;
|
||||
@ -116,7 +116,7 @@ public:
|
||||
return false;
|
||||
}
|
||||
|
||||
void checkSettingNameIsAllowed(const std::string_view & setting_name) const
|
||||
void checkSettingNameIsAllowed(std::string_view setting_name) const
|
||||
{
|
||||
if (isSettingNameAllowed(setting_name))
|
||||
return;
|
||||
@ -165,13 +165,12 @@ void AccessControl::setUpFromMainConfig(const Poco::Util::AbstractConfiguration
|
||||
setNoPasswordAllowed(config_.getBool("allow_no_password", true));
|
||||
setPlaintextPasswordAllowed(config_.getBool("allow_plaintext_password", true));
|
||||
|
||||
setEnabledUsersWithoutRowPoliciesCanReadRows(config_.getBool(
|
||||
"access_control_improvements.users_without_row_policies_can_read_rows",
|
||||
false /* false because we need to be compatible with earlier access configurations */));
|
||||
|
||||
setOnClusterQueriesRequireClusterGrant(config_.getBool(
|
||||
"access_control_improvements.on_cluster_queries_require_cluster_grant",
|
||||
false /* false because we need to be compatible with earlier access configurations */));
|
||||
/// Optional improvements in access control system.
|
||||
/// The default values are false because we need to be compatible with earlier access configurations
|
||||
setEnabledUsersWithoutRowPoliciesCanReadRows(config_.getBool("access_control_improvements.users_without_row_policies_can_read_rows", false));
|
||||
setOnClusterQueriesRequireClusterGrant(config_.getBool("access_control_improvements.on_cluster_queries_require_cluster_grant", false));
|
||||
setSelectFromSystemDatabaseRequiresGrant(config_.getBool("access_control_improvements.select_from_system_db_requires_grant", false));
|
||||
setSelectFromInformationSchemaRequiresGrant(config_.getBool("access_control_improvements.select_from_information_schema_requires_grant", false));
|
||||
|
||||
addStoragesFromMainConfig(config_, config_path_, get_zookeeper_function_);
|
||||
}
|
||||
|
||||
@ -152,6 +152,12 @@ public:
|
||||
void setOnClusterQueriesRequireClusterGrant(bool enable) { on_cluster_queries_require_cluster_grant = enable; }
|
||||
bool doesOnClusterQueriesRequireClusterGrant() const { return on_cluster_queries_require_cluster_grant; }
|
||||
|
||||
void setSelectFromSystemDatabaseRequiresGrant(bool enable) { select_from_system_db_requires_grant = enable; }
|
||||
bool doesSelectFromSystemDatabaseRequireGrant() const { return select_from_system_db_requires_grant; }
|
||||
|
||||
void setSelectFromInformationSchemaRequiresGrant(bool enable) { select_from_information_schema_requires_grant = enable; }
|
||||
bool doesSelectFromInformationSchemaRequireGrant() const { return select_from_information_schema_requires_grant; }
|
||||
|
||||
std::shared_ptr<const ContextAccess> getContextAccess(
|
||||
const UUID & user_id,
|
||||
const std::vector<UUID> & current_roles,
|
||||
@ -215,6 +221,8 @@ private:
|
||||
std::atomic_bool allow_no_password = true;
|
||||
std::atomic_bool users_without_row_policies_can_read_rows = false;
|
||||
std::atomic_bool on_cluster_queries_require_cluster_grant = false;
|
||||
std::atomic_bool select_from_system_db_requires_grant = false;
|
||||
std::atomic_bool select_from_information_schema_requires_grant = false;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -252,7 +252,7 @@ public:
|
||||
}
|
||||
|
||||
template <typename ... Args>
|
||||
void grant(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
|
||||
void grant(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
|
||||
{
|
||||
auto & child = getChild(name);
|
||||
child.grant(flags_, subnames...);
|
||||
@ -279,7 +279,7 @@ public:
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
void revoke(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames)
|
||||
void revoke(const AccessFlags & flags_, std::string_view name, const Args &... subnames)
|
||||
{
|
||||
auto & child = getChild(name);
|
||||
|
||||
@ -306,7 +306,7 @@ public:
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
bool isGranted(const AccessFlags & flags_, const std::string_view & name, const Args &... subnames) const
|
||||
bool isGranted(const AccessFlags & flags_, std::string_view name, const Args &... subnames) const
|
||||
{
|
||||
AccessFlags flags_to_check = flags_ - min_flags_with_children;
|
||||
if (!flags_to_check)
|
||||
@ -388,11 +388,11 @@ public:
|
||||
return res;
|
||||
}
|
||||
|
||||
void modifyFlags(const ModifyFlagsFunction & function, bool & flags_added, bool & flags_removed)
|
||||
void modifyFlags(const ModifyFlagsFunction & function, bool grant_option, bool & flags_added, bool & flags_removed)
|
||||
{
|
||||
flags_added = false;
|
||||
flags_removed = false;
|
||||
modifyFlagsRec(function, flags_added, flags_removed);
|
||||
modifyFlagsRec(function, grant_option, flags_added, flags_removed);
|
||||
if (flags_added || flags_removed)
|
||||
optimizeTree();
|
||||
}
|
||||
@ -415,7 +415,7 @@ private:
|
||||
AccessFlags getAllGrantableFlags() const { return ::DB::getAllGrantableFlags(level); }
|
||||
AccessFlags getChildAllGrantableFlags() const { return ::DB::getAllGrantableFlags(static_cast<Level>(level + 1)); }
|
||||
|
||||
Node * tryGetChild(const std::string_view & name) const
|
||||
Node * tryGetChild(std::string_view name) const
|
||||
{
|
||||
if (!children)
|
||||
return nullptr;
|
||||
@ -425,7 +425,7 @@ private:
|
||||
return &it->second;
|
||||
}
|
||||
|
||||
Node & getChild(const std::string_view & name)
|
||||
Node & getChild(std::string_view name)
|
||||
{
|
||||
auto * child = tryGetChild(name);
|
||||
if (child)
|
||||
@ -669,11 +669,11 @@ private:
|
||||
}
|
||||
|
||||
template <typename ... ParentNames>
|
||||
void modifyFlagsRec(const ModifyFlagsFunction & function, bool & flags_added, bool & flags_removed, const ParentNames & ... parent_names)
|
||||
void modifyFlagsRec(const ModifyFlagsFunction & function, bool grant_option, bool & flags_added, bool & flags_removed, const ParentNames & ... parent_names)
|
||||
{
|
||||
auto invoke = [&function](const AccessFlags & flags_, const AccessFlags & min_flags_with_children_, const AccessFlags & max_flags_with_children_, std::string_view database_ = {}, std::string_view table_ = {}, std::string_view column_ = {}) -> AccessFlags
|
||||
auto invoke = [function, grant_option](const AccessFlags & flags_, const AccessFlags & min_flags_with_children_, const AccessFlags & max_flags_with_children_, std::string_view database_ = {}, std::string_view table_ = {}, std::string_view column_ = {}) -> AccessFlags
|
||||
{
|
||||
return function(flags_, min_flags_with_children_, max_flags_with_children_, database_, table_, column_);
|
||||
return function(flags_, min_flags_with_children_, max_flags_with_children_, database_, table_, column_, grant_option);
|
||||
};
|
||||
|
||||
if constexpr (sizeof...(ParentNames) < 3)
|
||||
@ -683,7 +683,7 @@ private:
|
||||
for (auto & child : *children | boost::adaptors::map_values)
|
||||
{
|
||||
const String & child_name = *child.node_name;
|
||||
child.modifyFlagsRec(function, flags_added, flags_removed, parent_names..., child_name);
|
||||
child.modifyFlagsRec(function, grant_option, flags_added, flags_removed, parent_names..., child_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -819,20 +819,20 @@ void AccessRights::grantImpl(const AccessRightsElements & elements)
|
||||
}
|
||||
|
||||
void AccessRights::grant(const AccessFlags & flags) { grantImpl<false>(flags); }
|
||||
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl<false>(flags, database); }
|
||||
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<false>(flags, database, table); }
|
||||
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<false>(flags, database, table, column); }
|
||||
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::grant(const AccessFlags & flags, std::string_view database) { grantImpl<false>(flags, database); }
|
||||
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<false>(flags, database, table); }
|
||||
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<false>(flags, database, table, column); }
|
||||
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::grant(const AccessRightsElement & element) { grantImpl<false>(element); }
|
||||
void AccessRights::grant(const AccessRightsElements & elements) { grantImpl<false>(elements); }
|
||||
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl<true>(flags); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database) { grantImpl<true>(flags, database); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl<true>(flags, database, table); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl<true>(flags, database, table, column); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database) { grantImpl<true>(flags, database); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { grantImpl<true>(flags, database, table); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { grantImpl<true>(flags, database, table, column); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { grantImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { grantImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::grantWithGrantOption(const AccessRightsElement & element) { grantImpl<true>(element); }
|
||||
void AccessRights::grantWithGrantOption(const AccessRightsElements & elements) { grantImpl<true>(elements); }
|
||||
|
||||
@ -892,20 +892,20 @@ void AccessRights::revokeImpl(const AccessRightsElements & elements)
|
||||
}
|
||||
|
||||
void AccessRights::revoke(const AccessFlags & flags) { revokeImpl<false>(flags); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl<false>(flags, database); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<false>(flags, database, table); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<false>(flags, database, table, column); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, std::string_view database) { revokeImpl<false>(flags, database); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<false>(flags, database, table); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<false>(flags, database, table, column); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<false>(flags, database, table, columns); }
|
||||
void AccessRights::revoke(const AccessRightsElement & element) { revokeImpl<false>(element); }
|
||||
void AccessRights::revoke(const AccessRightsElements & elements) { revokeImpl<false>(elements); }
|
||||
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags) { revokeImpl<true>(flags); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database) { revokeImpl<true>(flags, database); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl<true>(flags, database, table); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<true>(flags, database, table, column); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database) { revokeImpl<true>(flags, database); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) { revokeImpl<true>(flags, database, table); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) { revokeImpl<true>(flags, database, table, column); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) { revokeImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) { revokeImpl<true>(flags, database, table, columns); }
|
||||
void AccessRights::revokeGrantOption(const AccessRightsElement & element) { revokeImpl<true>(element); }
|
||||
void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { revokeImpl<true>(elements); }
|
||||
|
||||
@ -984,20 +984,20 @@ bool AccessRights::isGrantedImpl(const AccessRightsElements & elements) const
|
||||
}
|
||||
|
||||
bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl<false>(flags); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<false>(flags, database); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<false>(flags, database, table); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<false>(flags, database, table, column); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<false>(flags, database); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<false>(flags, database, table); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<false>(flags, database, table, column); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
|
||||
bool AccessRights::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<false>(flags, database, table, columns); }
|
||||
bool AccessRights::isGranted(const AccessRightsElement & element) const { return isGrantedImpl<false>(element); }
|
||||
bool AccessRights::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl<false>(elements); }
|
||||
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl<true>(flags); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl<true>(flags, database); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl<true>(flags, database, table); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl<true>(flags, database, table, column); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return isGrantedImpl<true>(flags, database); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return isGrantedImpl<true>(flags, database, table); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return isGrantedImpl<true>(flags, database, table, column); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
|
||||
bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return isGrantedImpl<true>(flags, database, table, columns); }
|
||||
bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl<true>(element); }
|
||||
bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl<true>(elements); }
|
||||
|
||||
@ -1062,24 +1062,21 @@ void AccessRights::modifyFlags(const ModifyFlagsFunction & function)
|
||||
{
|
||||
if (!root)
|
||||
return;
|
||||
|
||||
bool flags_added, flags_removed;
|
||||
root->modifyFlags(function, flags_added, flags_removed);
|
||||
root->modifyFlags(function, false, flags_added, flags_removed);
|
||||
if (flags_removed && root_with_grant_option)
|
||||
root_with_grant_option->makeIntersection(*root);
|
||||
}
|
||||
|
||||
|
||||
void AccessRights::modifyFlagsWithGrantOption(const ModifyFlagsFunction & function)
|
||||
{
|
||||
if (!root_with_grant_option)
|
||||
return;
|
||||
bool flags_added, flags_removed;
|
||||
root_with_grant_option->modifyFlags(function, flags_added, flags_removed);
|
||||
if (flags_added)
|
||||
if (root_with_grant_option)
|
||||
{
|
||||
if (!root)
|
||||
root = std::make_unique<Node>();
|
||||
root->makeUnion(*root_with_grant_option);
|
||||
root_with_grant_option->modifyFlags(function, true, flags_added, flags_removed);
|
||||
if (flags_added)
|
||||
{
|
||||
if (!root)
|
||||
root = std::make_unique<Node>();
|
||||
root->makeUnion(*root_with_grant_option);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -39,59 +39,59 @@ public:
|
||||
/// Grants access on a specified database/table/column.
|
||||
/// Does nothing if the specified access has been already granted.
|
||||
void grant(const AccessFlags & flags);
|
||||
void grant(const AccessFlags & flags, const std::string_view & database);
|
||||
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
|
||||
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
|
||||
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
|
||||
void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
|
||||
void grant(const AccessFlags & flags, std::string_view database);
|
||||
void grant(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void grant(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void grant(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
void grant(const AccessRightsElement & element);
|
||||
void grant(const AccessRightsElements & elements);
|
||||
|
||||
void grantWithGrantOption(const AccessFlags & flags);
|
||||
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database);
|
||||
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
|
||||
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
|
||||
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
|
||||
void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void grantWithGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
void grantWithGrantOption(const AccessRightsElement & element);
|
||||
void grantWithGrantOption(const AccessRightsElements & elements);
|
||||
|
||||
/// Revokes a specified access granted earlier on a specified database/table/column.
|
||||
/// For example, revoke(AccessType::ALL) revokes all grants at all, just like clear();
|
||||
void revoke(const AccessFlags & flags);
|
||||
void revoke(const AccessFlags & flags, const std::string_view & database);
|
||||
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
|
||||
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
|
||||
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
|
||||
void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
|
||||
void revoke(const AccessFlags & flags, std::string_view database);
|
||||
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void revoke(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
void revoke(const AccessRightsElement & element);
|
||||
void revoke(const AccessRightsElements & elements);
|
||||
|
||||
void revokeGrantOption(const AccessFlags & flags);
|
||||
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database);
|
||||
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table);
|
||||
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column);
|
||||
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
|
||||
void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns);
|
||||
void revokeGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns);
|
||||
void revokeGrantOption(const AccessRightsElement & element);
|
||||
void revokeGrantOption(const AccessRightsElements & elements);
|
||||
|
||||
/// Whether a specified access granted.
|
||||
bool isGranted(const AccessFlags & flags) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool isGranted(const AccessRightsElement & element) const;
|
||||
bool isGranted(const AccessRightsElements & elements) const;
|
||||
|
||||
bool hasGrantOption(const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const AccessRightsElements & elements) const;
|
||||
|
||||
@ -107,11 +107,11 @@ public:
|
||||
const AccessFlags & flags,
|
||||
const AccessFlags & min_flags_with_children,
|
||||
const AccessFlags & max_flags_with_children,
|
||||
const std::string_view & database,
|
||||
const std::string_view & table,
|
||||
const std::string_view & column)>;
|
||||
std::string_view database,
|
||||
std::string_view table,
|
||||
std::string_view column,
|
||||
bool grant_option)>;
|
||||
void modifyFlags(const ModifyFlagsFunction & function);
|
||||
void modifyFlagsWithGrantOption(const ModifyFlagsFunction & function);
|
||||
|
||||
friend bool operator ==(const AccessRights & left, const AccessRights & right);
|
||||
friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); }
|
||||
|
||||
@ -26,17 +26,17 @@ namespace
|
||||
return (Util::stringToDigest(password) == password_plaintext);
|
||||
}
|
||||
|
||||
bool checkPasswordDoubleSHA1(const std::string_view & password, const Digest & password_double_sha1)
|
||||
bool checkPasswordDoubleSHA1(std::string_view password, const Digest & password_double_sha1)
|
||||
{
|
||||
return (Util::encodeDoubleSHA1(password) == password_double_sha1);
|
||||
}
|
||||
|
||||
bool checkPasswordSHA256(const std::string_view & password, const Digest & password_sha256, const String & salt)
|
||||
bool checkPasswordSHA256(std::string_view password, const Digest & password_sha256, const String & salt)
|
||||
{
|
||||
return Util::encodeSHA256(String(password).append(salt)) == password_sha256;
|
||||
}
|
||||
|
||||
bool checkPasswordDoubleSHA1MySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_double_sha1)
|
||||
bool checkPasswordDoubleSHA1MySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_double_sha1)
|
||||
{
|
||||
/// scrambled_password = SHA1(password) XOR SHA1(scramble <concat> SHA1(SHA1(password)))
|
||||
|
||||
@ -61,7 +61,7 @@ namespace
|
||||
return calculated_password_double_sha1 == password_double_sha1;
|
||||
}
|
||||
|
||||
bool checkPasswordPlainTextMySQL(const std::string_view & scramble, const std::string_view & scrambled_password, const Digest & password_plaintext)
|
||||
bool checkPasswordPlainTextMySQL(std::string_view scramble, std::string_view scrambled_password, const Digest & password_plaintext)
|
||||
{
|
||||
return checkPasswordDoubleSHA1MySQL(scramble, scrambled_password, Util::encodeDoubleSHA1(password_plaintext));
|
||||
}
|
||||
|
||||
@ -35,7 +35,7 @@ namespace
|
||||
return access_type_to_flags_mapping[static_cast<size_t>(type)];
|
||||
}
|
||||
|
||||
Flags keywordToFlags(const std::string_view & keyword) const
|
||||
Flags keywordToFlags(std::string_view keyword) const
|
||||
{
|
||||
auto it = keyword_to_flags_map.find(keyword);
|
||||
if (it == keyword_to_flags_map.end())
|
||||
@ -142,14 +142,14 @@ namespace
|
||||
}
|
||||
};
|
||||
|
||||
static String replaceUnderscoreWithSpace(const std::string_view & str)
|
||||
static String replaceUnderscoreWithSpace(std::string_view str)
|
||||
{
|
||||
String res{str};
|
||||
boost::replace_all(res, "_", " ");
|
||||
return res;
|
||||
}
|
||||
|
||||
static Strings splitAliases(const std::string_view & str)
|
||||
static Strings splitAliases(std::string_view str)
|
||||
{
|
||||
Strings aliases;
|
||||
boost::split(aliases, str, boost::is_any_of(","));
|
||||
@ -160,10 +160,10 @@ namespace
|
||||
|
||||
static void makeNode(
|
||||
AccessType access_type,
|
||||
const std::string_view & name,
|
||||
const std::string_view & aliases,
|
||||
std::string_view name,
|
||||
std::string_view aliases,
|
||||
NodeType node_type,
|
||||
const std::string_view & parent_group_name,
|
||||
std::string_view parent_group_name,
|
||||
std::unordered_map<std::string_view, Node *> & nodes,
|
||||
std::unordered_map<std::string_view, NodePtr> & owned_nodes,
|
||||
size_t & next_flag)
|
||||
@ -353,7 +353,7 @@ namespace
|
||||
|
||||
|
||||
AccessFlags::AccessFlags(AccessType type) : flags(Helper::instance().accessTypeToFlags(type)) {}
|
||||
AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
|
||||
AccessFlags::AccessFlags(std::string_view keyword) : flags(Helper::instance().keywordToFlags(keyword)) {}
|
||||
AccessFlags::AccessFlags(const std::vector<std::string_view> & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
|
||||
AccessFlags::AccessFlags(const Strings & keywords) : flags(Helper::instance().keywordsToFlags(keywords)) {}
|
||||
String AccessFlags::toString() const { return Helper::instance().flagsToString(flags); }
|
||||
|
||||
@ -21,7 +21,7 @@ public:
|
||||
AccessFlags() = default;
|
||||
|
||||
/// Constructs from a string like "SELECT".
|
||||
AccessFlags(const std::string_view & keyword); /// NOLINT
|
||||
AccessFlags(std::string_view keyword); /// NOLINT
|
||||
|
||||
/// Constructs from a list of strings like "SELECT, UPDATE, INSERT".
|
||||
AccessFlags(const std::vector<std::string_view> & keywords); /// NOLINT
|
||||
|
||||
@ -81,7 +81,7 @@ namespace
|
||||
}
|
||||
|
||||
bool need_comma = false;
|
||||
for (const std::string_view & keyword : keywords)
|
||||
for (std::string_view keyword : keywords)
|
||||
{
|
||||
if (need_comma)
|
||||
result.append(", ");
|
||||
@ -145,18 +145,18 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_)
|
||||
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_)
|
||||
: access_flags(access_flags_), database(database_), any_database(false)
|
||||
{
|
||||
}
|
||||
|
||||
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_)
|
||||
AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_)
|
||||
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false)
|
||||
{
|
||||
}
|
||||
|
||||
AccessRightsElement::AccessRightsElement(
|
||||
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_)
|
||||
AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_)
|
||||
: access_flags(access_flags_)
|
||||
, database(database_)
|
||||
, table(table_)
|
||||
@ -169,8 +169,8 @@ AccessRightsElement::AccessRightsElement(
|
||||
|
||||
AccessRightsElement::AccessRightsElement(
|
||||
AccessFlags access_flags_,
|
||||
const std::string_view & database_,
|
||||
const std::string_view & table_,
|
||||
std::string_view database_,
|
||||
std::string_view table_,
|
||||
const std::vector<std::string_view> & columns_)
|
||||
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false), any_column(false)
|
||||
{
|
||||
@ -180,7 +180,7 @@ AccessRightsElement::AccessRightsElement(
|
||||
}
|
||||
|
||||
AccessRightsElement::AccessRightsElement(
|
||||
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_)
|
||||
AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_)
|
||||
: access_flags(access_flags_)
|
||||
, database(database_)
|
||||
, table(table_)
|
||||
|
||||
@ -28,19 +28,19 @@ struct AccessRightsElement
|
||||
|
||||
explicit AccessRightsElement(AccessFlags access_flags_) : access_flags(access_flags_) {}
|
||||
|
||||
AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_);
|
||||
AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_);
|
||||
AccessRightsElement(AccessFlags access_flags_, std::string_view database_);
|
||||
AccessRightsElement(AccessFlags access_flags_, std::string_view database_, std::string_view table_);
|
||||
AccessRightsElement(
|
||||
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_);
|
||||
AccessFlags access_flags_, std::string_view database_, std::string_view table_, std::string_view column_);
|
||||
|
||||
AccessRightsElement(
|
||||
AccessFlags access_flags_,
|
||||
const std::string_view & database_,
|
||||
const std::string_view & table_,
|
||||
std::string_view database_,
|
||||
std::string_view table_,
|
||||
const std::vector<std::string_view> & columns_);
|
||||
|
||||
AccessRightsElement(
|
||||
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_);
|
||||
AccessFlags access_flags_, std::string_view database_, std::string_view table_, const Strings & columns_);
|
||||
|
||||
bool empty() const { return !access_flags || (!any_column && columns.empty()); }
|
||||
|
||||
|
||||
@ -35,7 +35,7 @@ namespace
|
||||
#undef ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING
|
||||
}
|
||||
|
||||
void addToMapping(AccessType type, const std::string_view & str)
|
||||
void addToMapping(AccessType type, std::string_view str)
|
||||
{
|
||||
String str2{str};
|
||||
boost::replace_all(str2, "_", " ");
|
||||
|
||||
@ -71,7 +71,7 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty
|
||||
}
|
||||
|
||||
|
||||
AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::string_view & text [[maybe_unused]])
|
||||
AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_view text [[maybe_unused]])
|
||||
{
|
||||
#if USE_SSL
|
||||
Digest hash;
|
||||
@ -86,7 +86,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(const std::str
|
||||
}
|
||||
|
||||
|
||||
AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(const std::string_view & text)
|
||||
AuthenticationData::Digest AuthenticationData::Util::encodeSHA1(std::string_view text)
|
||||
{
|
||||
Poco::SHA1Engine engine;
|
||||
engine.update(text.data(), text.size());
|
||||
|
||||
@ -96,11 +96,11 @@ public:
|
||||
|
||||
struct Util
|
||||
{
|
||||
static Digest stringToDigest(const std::string_view & text) { return Digest(text.data(), text.data() + text.size()); }
|
||||
static Digest encodeSHA256(const std::string_view & text);
|
||||
static Digest encodeSHA1(const std::string_view & text);
|
||||
static Digest stringToDigest(std::string_view text) { return Digest(text.data(), text.data() + text.size()); }
|
||||
static Digest encodeSHA256(std::string_view text);
|
||||
static Digest encodeSHA1(std::string_view text);
|
||||
static Digest encodeSHA1(const Digest & text) { return encodeSHA1(std::string_view{reinterpret_cast<const char *>(text.data()), text.size()}); }
|
||||
static Digest encodeDoubleSHA1(const std::string_view & text) { return encodeSHA1(encodeSHA1(text)); }
|
||||
static Digest encodeDoubleSHA1(std::string_view text) { return encodeSHA1(encodeSHA1(text)); }
|
||||
static Digest encodeDoubleSHA1(const Digest & text) { return encodeSHA1(encodeSHA1(text)); }
|
||||
};
|
||||
|
||||
|
||||
@ -44,9 +44,17 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
AccessRights addImplicitAccessRights(const AccessRights & access)
|
||||
AccessRights addImplicitAccessRights(const AccessRights & access, const AccessControl & access_control)
|
||||
{
|
||||
auto modifier = [&](const AccessFlags & flags, const AccessFlags & min_flags_with_children, const AccessFlags & max_flags_with_children, const std::string_view & database, const std::string_view & table, const std::string_view & column) -> AccessFlags
|
||||
AccessFlags max_flags;
|
||||
|
||||
auto modifier = [&](const AccessFlags & flags,
|
||||
const AccessFlags & min_flags_with_children,
|
||||
const AccessFlags & max_flags_with_children,
|
||||
std::string_view database,
|
||||
std::string_view table,
|
||||
std::string_view column,
|
||||
bool /* grant_option */) -> AccessFlags
|
||||
{
|
||||
size_t level = !database.empty() + !table.empty() + !column.empty();
|
||||
AccessFlags res = flags;
|
||||
@ -115,17 +123,80 @@ namespace
|
||||
res |= show_databases;
|
||||
}
|
||||
|
||||
max_flags |= res;
|
||||
|
||||
return res;
|
||||
};
|
||||
|
||||
AccessRights res = access;
|
||||
res.modifyFlags(modifier);
|
||||
res.modifyFlagsWithGrantOption(modifier);
|
||||
|
||||
/// Anyone has access to the "system" and "information_schema" database.
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA);
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE);
|
||||
/// If "select_from_system_db_requires_grant" is enabled we provide implicit grants only for a few tables in the system database.
|
||||
if (access_control.doesSelectFromSystemDatabaseRequireGrant())
|
||||
{
|
||||
const char * always_accessible_tables[] = {
|
||||
/// Constant tables
|
||||
"one",
|
||||
|
||||
/// "numbers", "numbers_mt", "zeros", "zeros_mt" were excluded because they can generate lots of values and
|
||||
/// that can decrease performance in some cases.
|
||||
|
||||
"contributors",
|
||||
"licenses",
|
||||
"time_zones",
|
||||
"collations",
|
||||
|
||||
"formats",
|
||||
"privileges",
|
||||
"data_type_families",
|
||||
"table_engines",
|
||||
"table_functions",
|
||||
"aggregate_function_combinators",
|
||||
|
||||
"functions", /// Can contain user-defined functions
|
||||
|
||||
/// The following tables hide some rows if the current user doesn't have corresponding SHOW privileges.
|
||||
"databases",
|
||||
"tables",
|
||||
"columns",
|
||||
|
||||
/// Specific to the current session
|
||||
"settings",
|
||||
"current_roles",
|
||||
"enabled_roles",
|
||||
"quota_usage"
|
||||
};
|
||||
|
||||
for (const auto * table_name : always_accessible_tables)
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, table_name);
|
||||
|
||||
if (max_flags.contains(AccessType::SHOW_USERS))
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "users");
|
||||
|
||||
if (max_flags.contains(AccessType::SHOW_ROLES))
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "roles");
|
||||
|
||||
if (max_flags.contains(AccessType::SHOW_ROW_POLICIES))
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "row_policies");
|
||||
|
||||
if (max_flags.contains(AccessType::SHOW_SETTINGS_PROFILES))
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "settings_profiles");
|
||||
|
||||
if (max_flags.contains(AccessType::SHOW_QUOTAS))
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "quotas");
|
||||
}
|
||||
else
|
||||
{
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE);
|
||||
}
|
||||
|
||||
/// If "select_from_information_schema_requires_grant" is enabled we don't provide implicit grants for the information_schema database.
|
||||
if (!access_control.doesSelectFromInformationSchemaRequireGrant())
|
||||
{
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA);
|
||||
res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -141,7 +212,7 @@ namespace
|
||||
std::string_view getDatabase() { return {}; }
|
||||
|
||||
template <typename... OtherArgs>
|
||||
std::string_view getDatabase(const std::string_view & arg1, const OtherArgs &...) { return arg1; }
|
||||
std::string_view getDatabase(std::string_view arg1, const OtherArgs &...) { return arg1; }
|
||||
}
|
||||
|
||||
|
||||
@ -247,7 +318,7 @@ void ContextAccess::setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> &
|
||||
void ContextAccess::calculateAccessRights() const
|
||||
{
|
||||
access = std::make_shared<AccessRights>(mixAccessRightsFromUserAndRoles(*user, *roles_info));
|
||||
access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*access));
|
||||
access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*access, *access_control));
|
||||
|
||||
if (trace_log)
|
||||
{
|
||||
@ -342,7 +413,7 @@ std::shared_ptr<const ContextAccess> ContextAccess::getFullAccess()
|
||||
auto full_access = std::shared_ptr<ContextAccess>(new ContextAccess);
|
||||
full_access->is_full_access = true;
|
||||
full_access->access = std::make_shared<AccessRights>(AccessRights::getFullAccess());
|
||||
full_access->access_with_implicit = std::make_shared<AccessRights>(addImplicitAccessRights(*full_access->access));
|
||||
full_access->access_with_implicit = full_access->access;
|
||||
return full_access;
|
||||
}();
|
||||
return res;
|
||||
@ -413,7 +484,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
};
|
||||
|
||||
if (is_full_access)
|
||||
return access_granted();
|
||||
return true;
|
||||
|
||||
if (user_was_dropped)
|
||||
return access_denied("User has been dropped", ErrorCodes::UNKNOWN_USER);
|
||||
@ -422,7 +493,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
flags &= ~AccessType::CLUSTER;
|
||||
|
||||
if (!flags)
|
||||
return access_granted();
|
||||
return true;
|
||||
|
||||
/// Access to temporary tables is controlled in an unusual way, not like normal tables.
|
||||
/// Creating of temporary tables is controlled by AccessType::CREATE_TEMPORARY_TABLES grant,
|
||||
@ -519,7 +590,7 @@ bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const
|
||||
bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(flags, database.empty() ? params.current_database : database, args...);
|
||||
}
|
||||
@ -564,38 +635,38 @@ bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
|
||||
}
|
||||
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl<false, false>(flags); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, false>(flags, database); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, false>(flags, database, table); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(flags, database); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(flags, database, table); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl<false, false>(element); }
|
||||
bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(elements); }
|
||||
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return checkAccessImpl<false, true>(flags); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl<false, true>(flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl<false, true>(flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return checkAccessImpl<false, true>(element); }
|
||||
bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(elements); }
|
||||
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<true, false>(flags); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, false>(flags, database); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, false>(flags, database, table); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, false>(flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(flags, database); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(flags, database, table); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<true, false>(element); }
|
||||
void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<true, false>(elements); }
|
||||
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<true, true>(flags); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl<true, true>(flags, database); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl<true, true>(flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl<true, true>(flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(flags, database); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<true, true>(element); }
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true, true>(elements); }
|
||||
|
||||
|
||||
@ -101,40 +101,40 @@ public:
|
||||
/// Checks if a specified access is granted, and throws an exception if not.
|
||||
/// Empty database means the current database.
|
||||
void checkAccess(const AccessFlags & flags) const;
|
||||
void checkAccess(const AccessFlags & flags, const std::string_view & database) const;
|
||||
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkAccess(const AccessRightsElement & element) const;
|
||||
void checkAccess(const AccessRightsElements & elements) const;
|
||||
|
||||
void checkGrantOption(const AccessFlags & flags) const;
|
||||
void checkGrantOption(const AccessFlags & flags, const std::string_view & database) const;
|
||||
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkGrantOption(const AccessRightsElement & element) const;
|
||||
void checkGrantOption(const AccessRightsElements & elements) const;
|
||||
|
||||
/// Checks if a specified access is granted, and returns false if not.
|
||||
/// Empty database means the current database.
|
||||
bool isGranted(const AccessFlags & flags) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool isGranted(const AccessRightsElement & element) const;
|
||||
bool isGranted(const AccessRightsElements & elements) const;
|
||||
|
||||
bool hasGrantOption(const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const AccessRightsElements & elements) const;
|
||||
|
||||
@ -180,7 +180,7 @@ private:
|
||||
bool checkAccessImpl(const AccessFlags & flags) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const;
|
||||
bool checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessRightsElement & element) const;
|
||||
|
||||
@ -36,12 +36,12 @@ void SettingsConstraints::clear()
|
||||
}
|
||||
|
||||
|
||||
void SettingsConstraints::setMinValue(const std::string_view & setting_name, const Field & min_value)
|
||||
void SettingsConstraints::setMinValue(std::string_view setting_name, const Field & min_value)
|
||||
{
|
||||
getConstraintRef(setting_name).min_value = Settings::castValueUtil(setting_name, min_value);
|
||||
}
|
||||
|
||||
Field SettingsConstraints::getMinValue(const std::string_view & setting_name) const
|
||||
Field SettingsConstraints::getMinValue(std::string_view setting_name) const
|
||||
{
|
||||
const auto * ptr = tryGetConstraint(setting_name);
|
||||
if (ptr)
|
||||
@ -51,12 +51,12 @@ Field SettingsConstraints::getMinValue(const std::string_view & setting_name) co
|
||||
}
|
||||
|
||||
|
||||
void SettingsConstraints::setMaxValue(const std::string_view & setting_name, const Field & max_value)
|
||||
void SettingsConstraints::setMaxValue(std::string_view setting_name, const Field & max_value)
|
||||
{
|
||||
getConstraintRef(setting_name).max_value = Settings::castValueUtil(setting_name, max_value);
|
||||
}
|
||||
|
||||
Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) const
|
||||
Field SettingsConstraints::getMaxValue(std::string_view setting_name) const
|
||||
{
|
||||
const auto * ptr = tryGetConstraint(setting_name);
|
||||
if (ptr)
|
||||
@ -66,12 +66,12 @@ Field SettingsConstraints::getMaxValue(const std::string_view & setting_name) co
|
||||
}
|
||||
|
||||
|
||||
void SettingsConstraints::setReadOnly(const std::string_view & setting_name, bool read_only)
|
||||
void SettingsConstraints::setReadOnly(std::string_view setting_name, bool read_only)
|
||||
{
|
||||
getConstraintRef(setting_name).read_only = read_only;
|
||||
}
|
||||
|
||||
bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) const
|
||||
bool SettingsConstraints::isReadOnly(std::string_view setting_name) const
|
||||
{
|
||||
const auto * ptr = tryGetConstraint(setting_name);
|
||||
if (ptr)
|
||||
@ -81,7 +81,7 @@ bool SettingsConstraints::isReadOnly(const std::string_view & setting_name) cons
|
||||
}
|
||||
|
||||
|
||||
void SettingsConstraints::set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only)
|
||||
void SettingsConstraints::set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only)
|
||||
{
|
||||
auto & ref = getConstraintRef(setting_name);
|
||||
ref.min_value = Settings::castValueUtil(setting_name, min_value);
|
||||
@ -89,7 +89,7 @@ void SettingsConstraints::set(const std::string_view & setting_name, const Field
|
||||
ref.read_only = read_only;
|
||||
}
|
||||
|
||||
void SettingsConstraints::get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const
|
||||
void SettingsConstraints::get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const
|
||||
{
|
||||
const auto * ptr = tryGetConstraint(setting_name);
|
||||
if (ptr)
|
||||
@ -318,7 +318,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh
|
||||
}
|
||||
|
||||
|
||||
SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const std::string_view & setting_name)
|
||||
SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(std::string_view setting_name)
|
||||
{
|
||||
auto it = constraints.find(setting_name);
|
||||
if (it == constraints.end())
|
||||
@ -331,7 +331,7 @@ SettingsConstraints::Constraint & SettingsConstraints::getConstraintRef(const st
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(const std::string_view & setting_name) const
|
||||
const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(std::string_view setting_name) const
|
||||
{
|
||||
auto it = constraints.find(setting_name);
|
||||
if (it == constraints.end())
|
||||
|
||||
@ -61,17 +61,17 @@ public:
|
||||
void clear();
|
||||
bool empty() const { return constraints.empty(); }
|
||||
|
||||
void setMinValue(const std::string_view & setting_name, const Field & min_value);
|
||||
Field getMinValue(const std::string_view & setting_name) const;
|
||||
void setMinValue(std::string_view setting_name, const Field & min_value);
|
||||
Field getMinValue(std::string_view setting_name) const;
|
||||
|
||||
void setMaxValue(const std::string_view & setting_name, const Field & max_value);
|
||||
Field getMaxValue(const std::string_view & setting_name) const;
|
||||
void setMaxValue(std::string_view setting_name, const Field & max_value);
|
||||
Field getMaxValue(std::string_view setting_name) const;
|
||||
|
||||
void setReadOnly(const std::string_view & setting_name, bool read_only);
|
||||
bool isReadOnly(const std::string_view & setting_name) const;
|
||||
void setReadOnly(std::string_view setting_name, bool read_only);
|
||||
bool isReadOnly(std::string_view setting_name) const;
|
||||
|
||||
void set(const std::string_view & setting_name, const Field & min_value, const Field & max_value, bool read_only);
|
||||
void get(const std::string_view & setting_name, Field & min_value, Field & max_value, bool & read_only) const;
|
||||
void set(std::string_view setting_name, const Field & min_value, const Field & max_value, bool read_only);
|
||||
void get(std::string_view setting_name, Field & min_value, Field & max_value, bool & read_only) const;
|
||||
|
||||
void merge(const SettingsConstraints & other);
|
||||
|
||||
@ -105,8 +105,8 @@ private:
|
||||
};
|
||||
bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const;
|
||||
|
||||
Constraint & getConstraintRef(const std::string_view & setting_name);
|
||||
const Constraint * tryGetConstraint(const std::string_view & setting_name) const;
|
||||
Constraint & getConstraintRef(std::string_view setting_name);
|
||||
const Constraint * tryGetConstraint(std::string_view setting_name) const;
|
||||
|
||||
std::unordered_map<std::string_view, Constraint> constraints;
|
||||
const AccessControl * access_control = nullptr;
|
||||
|
||||
@ -132,7 +132,7 @@ public:
|
||||
key_ref = assert_cast<const ColumnString &>(key_column).getDataAt(offset + i);
|
||||
|
||||
#ifdef __cpp_lib_generic_unordered_lookup
|
||||
key = static_cast<std::string_view>(key_ref);
|
||||
key = key_ref.toView();
|
||||
#else
|
||||
key = key_ref.toString();
|
||||
#endif
|
||||
|
||||
@ -43,7 +43,7 @@ public:
|
||||
~ThetaSketchData() = default;
|
||||
|
||||
/// Insert original value without hash, as `datasketches::update_theta_sketch.update` will do the hash internal.
|
||||
void insertOriginal(const StringRef & value)
|
||||
void insertOriginal(StringRef value)
|
||||
{
|
||||
getSkUpdate()->update(value.data, value.size);
|
||||
}
|
||||
|
||||
@ -25,7 +25,6 @@ public:
|
||||
struct CreateParams
|
||||
{
|
||||
OpenMode open_mode = OpenMode::WRITE;
|
||||
std::optional<UUID> backup_uuid;
|
||||
BackupInfo backup_info;
|
||||
std::optional<BackupInfo> base_backup_info;
|
||||
String compression_method;
|
||||
@ -34,6 +33,7 @@ public:
|
||||
ContextPtr context;
|
||||
bool is_internal_backup = false;
|
||||
std::shared_ptr<IBackupCoordination> backup_coordination;
|
||||
std::optional<UUID> backup_uuid;
|
||||
};
|
||||
|
||||
static BackupFactory & instance();
|
||||
|
||||
@ -23,8 +23,9 @@ class IBackupWriter /// BackupWriterFile, BackupWriterDisk, BackupWriterS3
|
||||
public:
|
||||
virtual ~IBackupWriter() = default;
|
||||
virtual bool fileExists(const String & file_name) = 0;
|
||||
virtual bool fileContentsEqual(const String & file_name, const String & expected_file_contents) = 0;
|
||||
virtual std::unique_ptr<WriteBuffer> writeFile(const String & file_name) = 0;
|
||||
virtual void removeFilesAfterFailure(const Strings & file_names) = 0;
|
||||
virtual void removeFiles(const Strings & file_names) = 0;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -38,6 +38,25 @@ bool BackupWriterDisk::fileExists(const String & file_name)
|
||||
return disk->exists(path / file_name);
|
||||
}
|
||||
|
||||
bool BackupWriterDisk::fileContentsEqual(const String & file_name, const String & expected_file_contents)
|
||||
{
|
||||
if (!disk->exists(path / file_name))
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
auto in = disk->readFile(path / file_name);
|
||||
String actual_file_contents(expected_file_contents.size(), ' ');
|
||||
return (in->read(actual_file_contents.data(), actual_file_contents.size()) == actual_file_contents.size())
|
||||
&& (actual_file_contents == expected_file_contents) && in->eof();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<WriteBuffer> BackupWriterDisk::writeFile(const String & file_name)
|
||||
{
|
||||
auto file_path = path / file_name;
|
||||
@ -45,7 +64,7 @@ std::unique_ptr<WriteBuffer> BackupWriterDisk::writeFile(const String & file_nam
|
||||
return disk->writeFile(file_path);
|
||||
}
|
||||
|
||||
void BackupWriterDisk::removeFilesAfterFailure(const Strings & file_names)
|
||||
void BackupWriterDisk::removeFiles(const Strings & file_names)
|
||||
{
|
||||
for (const auto & file_name : file_names)
|
||||
disk->removeFileIfExists(path / file_name);
|
||||
|
||||
@ -30,8 +30,9 @@ public:
|
||||
~BackupWriterDisk() override;
|
||||
|
||||
bool fileExists(const String & file_name) override;
|
||||
bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
|
||||
std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
|
||||
void removeFilesAfterFailure(const Strings & file_names) override;
|
||||
void removeFiles(const Strings & file_names) override;
|
||||
|
||||
private:
|
||||
DiskPtr disk;
|
||||
|
||||
@ -39,6 +39,25 @@ bool BackupWriterFile::fileExists(const String & file_name)
|
||||
return fs::exists(path / file_name);
|
||||
}
|
||||
|
||||
bool BackupWriterFile::fileContentsEqual(const String & file_name, const String & expected_file_contents)
|
||||
{
|
||||
if (!fs::exists(path / file_name))
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
auto in = createReadBufferFromFileBase(path / file_name, {});
|
||||
String actual_file_contents(expected_file_contents.size(), ' ');
|
||||
return (in->read(actual_file_contents.data(), actual_file_contents.size()) == actual_file_contents.size())
|
||||
&& (actual_file_contents == expected_file_contents) && in->eof();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<WriteBuffer> BackupWriterFile::writeFile(const String & file_name)
|
||||
{
|
||||
auto file_path = path / file_name;
|
||||
@ -46,7 +65,7 @@ std::unique_ptr<WriteBuffer> BackupWriterFile::writeFile(const String & file_nam
|
||||
return std::make_unique<WriteBufferFromFile>(file_path);
|
||||
}
|
||||
|
||||
void BackupWriterFile::removeFilesAfterFailure(const Strings & file_names)
|
||||
void BackupWriterFile::removeFiles(const Strings & file_names)
|
||||
{
|
||||
for (const auto & file_name : file_names)
|
||||
fs::remove(path / file_name);
|
||||
|
||||
@ -27,8 +27,9 @@ public:
|
||||
~BackupWriterFile() override;
|
||||
|
||||
bool fileExists(const String & file_name) override;
|
||||
bool fileContentsEqual(const String & file_name, const String & expected_file_contents) override;
|
||||
std::unique_ptr<WriteBuffer> writeFile(const String & file_name) override;
|
||||
void removeFilesAfterFailure(const Strings & file_names) override;
|
||||
void removeFiles(const Strings & file_names) override;
|
||||
|
||||
private:
|
||||
std::filesystem::path path;
|
||||
|
||||
@ -37,6 +37,7 @@ namespace ErrorCodes
|
||||
extern const int BACKUP_ENTRY_ALREADY_EXISTS;
|
||||
extern const int BACKUP_ENTRY_NOT_FOUND;
|
||||
extern const int BACKUP_IS_EMPTY;
|
||||
extern const int FAILED_TO_SYNC_BACKUP_OR_RESTORE;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
@ -146,9 +147,9 @@ BackupImpl::BackupImpl(
|
||||
const std::optional<BackupInfo> & base_backup_info_,
|
||||
std::shared_ptr<IBackupWriter> writer_,
|
||||
const ContextPtr & context_,
|
||||
const std::optional<UUID> & backup_uuid_,
|
||||
bool is_internal_backup_,
|
||||
const std::shared_ptr<IBackupCoordination> & coordination_)
|
||||
const std::shared_ptr<IBackupCoordination> & coordination_,
|
||||
const std::optional<UUID> & backup_uuid_)
|
||||
: backup_name(backup_name_)
|
||||
, archive_params(archive_params_)
|
||||
, use_archives(!archive_params.archive_name.empty())
|
||||
@ -177,42 +178,28 @@ BackupImpl::~BackupImpl()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void BackupImpl::open(const ContextPtr & context)
|
||||
{
|
||||
std::lock_guard lock{mutex};
|
||||
|
||||
String file_name_to_check_existence;
|
||||
if (use_archives)
|
||||
file_name_to_check_existence = archive_params.archive_name;
|
||||
else
|
||||
file_name_to_check_existence = ".backup";
|
||||
bool backup_exists = (open_mode == OpenMode::WRITE) ? writer->fileExists(file_name_to_check_existence) : reader->fileExists(file_name_to_check_existence);
|
||||
|
||||
if (open_mode == OpenMode::WRITE)
|
||||
{
|
||||
if (backup_exists)
|
||||
throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "Backup {} already exists", backup_name);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!backup_exists)
|
||||
throw Exception(ErrorCodes::BACKUP_NOT_FOUND, "Backup {} not found", backup_name);
|
||||
}
|
||||
|
||||
if (open_mode == OpenMode::WRITE)
|
||||
{
|
||||
timestamp = std::time(nullptr);
|
||||
if (!uuid)
|
||||
uuid = UUIDHelpers::generateV4();
|
||||
lock_file_name = use_archives ? (archive_params.archive_name + ".lock") : ".lock";
|
||||
writing_finalized = false;
|
||||
|
||||
/// Check that we can write a backup there and create the lock file to own this destination.
|
||||
checkBackupDoesntExist();
|
||||
if (!is_internal_backup)
|
||||
createLockFile();
|
||||
checkLockFile(true);
|
||||
}
|
||||
|
||||
if (open_mode == OpenMode::READ)
|
||||
readBackupMetadata();
|
||||
|
||||
assert(uuid); /// Backup's UUID must be loaded or generated at this point.
|
||||
|
||||
if (base_backup_info)
|
||||
{
|
||||
BackupFactory::CreateParams params;
|
||||
@ -253,6 +240,8 @@ time_t BackupImpl::getTimestamp() const
|
||||
|
||||
void BackupImpl::writeBackupMetadata()
|
||||
{
|
||||
assert(!is_internal_backup);
|
||||
|
||||
Poco::AutoPtr<Poco::Util::XMLConfiguration> config{new Poco::Util::XMLConfiguration()};
|
||||
config->setUInt("version", CURRENT_BACKUP_VERSION);
|
||||
config->setString("timestamp", toString(LocalDateTime{timestamp}));
|
||||
@ -308,6 +297,8 @@ void BackupImpl::writeBackupMetadata()
|
||||
config->save(stream);
|
||||
String str = stream.str();
|
||||
|
||||
checkLockFile(true);
|
||||
|
||||
std::unique_ptr<WriteBuffer> out;
|
||||
if (use_archives)
|
||||
out = getArchiveWriter("")->writeFile(".backup");
|
||||
@ -321,9 +312,17 @@ void BackupImpl::readBackupMetadata()
|
||||
{
|
||||
std::unique_ptr<ReadBuffer> in;
|
||||
if (use_archives)
|
||||
{
|
||||
if (!reader->fileExists(archive_params.archive_name))
|
||||
throw Exception(ErrorCodes::BACKUP_NOT_FOUND, "Backup {} not found", backup_name);
|
||||
in = getArchiveReader("")->readFile(".backup");
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!reader->fileExists(".backup"))
|
||||
throw Exception(ErrorCodes::BACKUP_NOT_FOUND, "Backup {} not found", backup_name);
|
||||
in = reader->readFile(".backup");
|
||||
}
|
||||
|
||||
String str;
|
||||
readStringUntilEOF(str, *in);
|
||||
@ -387,6 +386,59 @@ void BackupImpl::readBackupMetadata()
|
||||
}
|
||||
}
|
||||
|
||||
void BackupImpl::checkBackupDoesntExist() const
|
||||
{
|
||||
String file_name_to_check_existence;
|
||||
if (use_archives)
|
||||
file_name_to_check_existence = archive_params.archive_name;
|
||||
else
|
||||
file_name_to_check_existence = ".backup";
|
||||
|
||||
if (writer->fileExists(file_name_to_check_existence))
|
||||
throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "Backup {} already exists", backup_name);
|
||||
|
||||
/// Check that no other backup (excluding internal backups) is writing to the same destination.
|
||||
if (!is_internal_backup)
|
||||
{
|
||||
assert(!lock_file_name.empty());
|
||||
if (writer->fileExists(lock_file_name))
|
||||
throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "Backup {} is being written already", backup_name);
|
||||
}
|
||||
}
|
||||
|
||||
void BackupImpl::createLockFile()
|
||||
{
|
||||
/// Internal backup must not create the lock file (it should be created by the initiator).
|
||||
assert(!is_internal_backup);
|
||||
|
||||
assert(uuid);
|
||||
auto out = writer->writeFile(lock_file_name);
|
||||
writeUUIDText(*uuid, *out);
|
||||
}
|
||||
|
||||
bool BackupImpl::checkLockFile(bool throw_if_failed) const
|
||||
{
|
||||
if (!lock_file_name.empty() && uuid && writer->fileContentsEqual(lock_file_name, toString(*uuid)))
|
||||
return true;
|
||||
|
||||
if (throw_if_failed)
|
||||
{
|
||||
if (!writer->fileExists(lock_file_name))
|
||||
throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "Lock file {} suddenly disappeared while writing backup {}", lock_file_name, backup_name);
|
||||
throw Exception(ErrorCodes::BACKUP_ALREADY_EXISTS, "A concurrent backup writing to the same destination {} detected", backup_name);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void BackupImpl::removeLockFile()
|
||||
{
|
||||
if (is_internal_backup)
|
||||
return; /// Internal backup must not remove the lock file (it's still used by the initiator).
|
||||
|
||||
if (checkLockFile(false))
|
||||
writer->removeFiles({lock_file_name});
|
||||
}
|
||||
|
||||
Strings BackupImpl::listFiles(const String & directory, bool recursive) const
|
||||
{
|
||||
std::lock_guard lock{mutex};
|
||||
@ -648,6 +700,9 @@ void BackupImpl::writeFile(const String & file_name, BackupEntryPtr entry)
|
||||
read_buffer = entry->getReadBuffer();
|
||||
read_buffer->seek(copy_pos, SEEK_SET);
|
||||
|
||||
if (!num_files_written)
|
||||
checkLockFile(true);
|
||||
|
||||
/// Copy the entry's data after `copy_pos`.
|
||||
std::unique_ptr<WriteBuffer> out;
|
||||
if (use_archives)
|
||||
@ -675,6 +730,7 @@ void BackupImpl::writeFile(const String & file_name, BackupEntryPtr entry)
|
||||
|
||||
copyData(*read_buffer, *out);
|
||||
out->finalize();
|
||||
++num_files_written;
|
||||
}
|
||||
|
||||
|
||||
@ -694,6 +750,7 @@ void BackupImpl::finalizeWriting()
|
||||
{
|
||||
LOG_TRACE(log, "Finalizing backup {}", backup_name);
|
||||
writeBackupMetadata();
|
||||
removeLockFile();
|
||||
LOG_TRACE(log, "Finalized backup {}", backup_name);
|
||||
}
|
||||
|
||||
@ -741,6 +798,9 @@ std::shared_ptr<IArchiveWriter> BackupImpl::getArchiveWriter(const String & suff
|
||||
|
||||
void BackupImpl::removeAllFilesAfterFailure()
|
||||
{
|
||||
if (is_internal_backup)
|
||||
return; /// Let the initiator remove unnecessary files.
|
||||
|
||||
try
|
||||
{
|
||||
LOG_INFO(log, "Removing all files of backup {} after failure", backup_name);
|
||||
@ -762,7 +822,11 @@ void BackupImpl::removeAllFilesAfterFailure()
|
||||
files_to_remove.push_back(file_info.data_file_name);
|
||||
}
|
||||
|
||||
writer->removeFilesAfterFailure(files_to_remove);
|
||||
if (!checkLockFile(false))
|
||||
return;
|
||||
|
||||
writer->removeFiles(files_to_remove);
|
||||
removeLockFile();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
|
||||
@ -47,9 +47,9 @@ public:
|
||||
const std::optional<BackupInfo> & base_backup_info_,
|
||||
std::shared_ptr<IBackupWriter> writer_,
|
||||
const ContextPtr & context_,
|
||||
const std::optional<UUID> & backup_uuid_ = {},
|
||||
bool is_internal_backup_ = false,
|
||||
const std::shared_ptr<IBackupCoordination> & coordination_ = {});
|
||||
const std::shared_ptr<IBackupCoordination> & coordination_ = {},
|
||||
const std::optional<UUID> & backup_uuid_ = {});
|
||||
|
||||
~BackupImpl() override;
|
||||
|
||||
@ -76,12 +76,25 @@ private:
|
||||
|
||||
void open(const ContextPtr & context);
|
||||
void close();
|
||||
|
||||
/// Writes the file ".backup" containing backup's metadata.
|
||||
void writeBackupMetadata();
|
||||
void readBackupMetadata();
|
||||
|
||||
/// Checks that a new backup doesn't exist yet.
|
||||
void checkBackupDoesntExist() const;
|
||||
|
||||
/// Lock file named ".lock" and containing the UUID of a backup is used to own the place where we're writing the backup.
|
||||
/// Thus it will not be allowed to put any other backup to the same place (even if the BACKUP command is executed on a different node).
|
||||
void createLockFile();
|
||||
bool checkLockFile(bool throw_if_failed) const;
|
||||
void removeLockFile();
|
||||
|
||||
void removeAllFilesAfterFailure();
|
||||
|
||||
String getArchiveNameWithSuffix(const String & suffix) const;
|
||||
std::shared_ptr<IArchiveReader> getArchiveReader(const String & suffix) const;
|
||||
std::shared_ptr<IArchiveWriter> getArchiveWriter(const String & suffix);
|
||||
void removeAllFilesAfterFailure();
|
||||
|
||||
const String backup_name;
|
||||
const ArchiveParams archive_params;
|
||||
@ -102,6 +115,8 @@ private:
|
||||
mutable std::unordered_map<String /* archive_suffix */, std::shared_ptr<IArchiveReader>> archive_readers;
|
||||
std::pair<String, std::shared_ptr<IArchiveWriter>> archive_writers[2];
|
||||
String current_archive_suffix;
|
||||
String lock_file_name;
|
||||
size_t num_files_written = 0;
|
||||
bool writing_finalized = false;
|
||||
const Poco::Logger * log;
|
||||
};
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -15,6 +16,48 @@ namespace ErrorCodes
|
||||
extern const int WRONG_BACKUP_SETTINGS;
|
||||
}
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
struct SettingFieldOptionalUUID
|
||||
{
|
||||
std::optional<UUID> value;
|
||||
|
||||
explicit SettingFieldOptionalUUID(const std::optional<UUID> & value_) : value(value_) {}
|
||||
|
||||
explicit SettingFieldOptionalUUID(const Field & field)
|
||||
{
|
||||
if (field.getType() == Field::Types::Null)
|
||||
{
|
||||
value = std::nullopt;
|
||||
return;
|
||||
}
|
||||
|
||||
if (field.getType() == Field::Types::String)
|
||||
{
|
||||
const String & str = field.get<const String &>();
|
||||
if (str.empty())
|
||||
{
|
||||
value = std::nullopt;
|
||||
return;
|
||||
}
|
||||
|
||||
UUID id;
|
||||
if (tryParse(id, str))
|
||||
{
|
||||
value = id;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
throw Exception(ErrorCodes::CANNOT_PARSE_BACKUP_SETTINGS, "Cannot parse uuid from {}", field);
|
||||
}
|
||||
|
||||
explicit operator Field() const { return Field(value ? toString(*value) : ""); }
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/// List of backup settings except base_backup_name and cluster_host_ids.
|
||||
#define LIST_OF_BACKUP_SETTINGS(M) \
|
||||
M(String, compression_method) \
|
||||
@ -26,7 +69,8 @@ namespace ErrorCodes
|
||||
M(UInt64, replica_num) \
|
||||
M(Bool, internal) \
|
||||
M(String, host_id) \
|
||||
M(String, coordination_zk_path)
|
||||
M(String, coordination_zk_path) \
|
||||
M(OptionalUUID, backup_uuid)
|
||||
|
||||
BackupSettings BackupSettings::fromBackupQuery(const ASTBackupQuery & query)
|
||||
{
|
||||
|
||||
@ -53,6 +53,10 @@ struct BackupSettings
|
||||
/// Path in Zookeeper used to coordinate a distributed backup created by BACKUP ON CLUSTER.
|
||||
String coordination_zk_path;
|
||||
|
||||
/// Internal, should not be specified by user.
|
||||
/// UUID of the backup. If it's not set it will be generated randomly.
|
||||
std::optional<UUID> backup_uuid;
|
||||
|
||||
static BackupSettings fromBackupQuery(const ASTBackupQuery & query);
|
||||
void copySettingsToQuery(ASTBackupQuery & query) const;
|
||||
|
||||
|
||||
@ -72,12 +72,15 @@ UUID BackupsWorker::start(const ASTPtr & backup_or_restore_query, ContextMutable
|
||||
|
||||
UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & context)
|
||||
{
|
||||
UUID backup_uuid = UUIDHelpers::generateV4();
|
||||
auto backup_query = std::static_pointer_cast<ASTBackupQuery>(query->clone());
|
||||
auto backup_settings = BackupSettings::fromBackupQuery(*backup_query);
|
||||
auto backup_info = BackupInfo::fromAST(*backup_query->backup_name);
|
||||
bool on_cluster = !backup_query->cluster.empty();
|
||||
|
||||
if (!backup_settings.backup_uuid)
|
||||
backup_settings.backup_uuid = UUIDHelpers::generateV4();
|
||||
UUID backup_uuid = *backup_settings.backup_uuid;
|
||||
|
||||
/// Prepare context to use.
|
||||
ContextPtr context_in_use = context;
|
||||
ContextMutablePtr mutable_context;
|
||||
@ -107,7 +110,7 @@ UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & c
|
||||
{
|
||||
if (async)
|
||||
{
|
||||
query_scope.emplace(context_in_use);
|
||||
query_scope.emplace(mutable_context);
|
||||
setThreadName("BackupWorker");
|
||||
}
|
||||
|
||||
@ -151,9 +154,9 @@ UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & c
|
||||
backup_create_params.compression_method = backup_settings.compression_method;
|
||||
backup_create_params.compression_level = backup_settings.compression_level;
|
||||
backup_create_params.password = backup_settings.password;
|
||||
backup_create_params.backup_uuid = backup_uuid;
|
||||
backup_create_params.is_internal_backup = backup_settings.internal;
|
||||
backup_create_params.backup_coordination = backup_coordination;
|
||||
backup_create_params.backup_uuid = backup_uuid;
|
||||
BackupMutablePtr backup = BackupFactory::instance().createBackup(backup_create_params);
|
||||
|
||||
/// Write the backup.
|
||||
|
||||
@ -180,7 +180,7 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
|
||||
writer = std::make_shared<BackupWriterFile>(path);
|
||||
else
|
||||
writer = std::make_shared<BackupWriterDisk>(disk, path);
|
||||
return std::make_unique<BackupImpl>(backup_name, archive_params, params.base_backup_info, writer, params.context, params.backup_uuid, params.is_internal_backup, params.backup_coordination);
|
||||
return std::make_unique<BackupImpl>(backup_name, archive_params, params.base_backup_info, writer, params.context, params.is_internal_backup, params.backup_coordination, params.backup_uuid);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -401,11 +401,11 @@ dbms_target_link_libraries (
|
||||
clickhouse_parsers
|
||||
ch_contrib::lz4
|
||||
Poco::JSON
|
||||
Poco::MongoDB
|
||||
string_utils
|
||||
PUBLIC
|
||||
boost::system
|
||||
clickhouse_common_io
|
||||
Poco::MongoDB
|
||||
)
|
||||
|
||||
if (TARGET ch::mysqlxx)
|
||||
|
||||
@ -601,6 +601,7 @@ void ClientBase::initLogsOutputStream()
|
||||
{
|
||||
WriteBuffer * wb = out_logs_buf.get();
|
||||
|
||||
bool color_logs = false;
|
||||
if (!out_logs_buf)
|
||||
{
|
||||
if (server_logs_file.empty())
|
||||
@ -608,11 +609,13 @@ void ClientBase::initLogsOutputStream()
|
||||
/// Use stderr by default
|
||||
out_logs_buf = std::make_unique<WriteBufferFromFileDescriptor>(STDERR_FILENO);
|
||||
wb = out_logs_buf.get();
|
||||
color_logs = stderr_is_a_tty;
|
||||
}
|
||||
else if (server_logs_file == "-")
|
||||
{
|
||||
/// Use stdout if --server_logs_file=- specified
|
||||
wb = &std_out;
|
||||
color_logs = stdout_is_a_tty;
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -622,7 +625,7 @@ void ClientBase::initLogsOutputStream()
|
||||
}
|
||||
}
|
||||
|
||||
logs_out_stream = std::make_unique<InternalTextLogs>(*wb, stdout_is_a_tty);
|
||||
logs_out_stream = std::make_unique<InternalTextLogs>(*wb, color_logs);
|
||||
}
|
||||
}
|
||||
|
||||
@ -980,8 +983,7 @@ void ClientBase::onProfileEvents(Block & block)
|
||||
else if (event_name == MemoryTracker::USAGE_EVENT_NAME)
|
||||
thread_times[host_name][thread_id].memory_usage = value;
|
||||
}
|
||||
auto elapsed_time = profile_events.watch.elapsedMicroseconds();
|
||||
progress_indication.updateThreadEventData(thread_times, elapsed_time);
|
||||
progress_indication.updateThreadEventData(thread_times);
|
||||
|
||||
if (need_render_progress)
|
||||
progress_indication.writeProgress();
|
||||
@ -2155,6 +2157,7 @@ void ClientBase::init(int argc, char ** argv)
|
||||
|
||||
stdin_is_a_tty = isatty(STDIN_FILENO);
|
||||
stdout_is_a_tty = isatty(STDOUT_FILENO);
|
||||
stderr_is_a_tty = isatty(STDERR_FILENO);
|
||||
terminal_width = getTerminalWidth();
|
||||
|
||||
Arguments common_arguments{""}; /// 0th argument is ignored.
|
||||
|
||||
@ -173,6 +173,7 @@ protected:
|
||||
|
||||
bool stdin_is_a_tty = false; /// stdin is a terminal.
|
||||
bool stdout_is_a_tty = false; /// stdout is a terminal.
|
||||
bool stderr_is_a_tty = false; /// stderr is a terminal.
|
||||
uint64_t terminal_width = 0;
|
||||
|
||||
ServerConnectionPtr connection;
|
||||
|
||||
@ -50,52 +50,58 @@ static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggesti
|
||||
{
|
||||
/// NOTE: Once you will update the completion list,
|
||||
/// do not forget to update 01676_clickhouse_client_autocomplete.sh
|
||||
WriteBufferFromOwnString query;
|
||||
query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM ("
|
||||
"SELECT name FROM system.functions"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.table_engines"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.formats"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.table_functions"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.data_type_families"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.merge_tree_settings"
|
||||
" UNION ALL "
|
||||
"SELECT name FROM system.settings"
|
||||
" UNION ALL ";
|
||||
String query;
|
||||
|
||||
auto add_subquery = [&](std::string_view select, std::string_view result_column_name)
|
||||
{
|
||||
if (!query.empty())
|
||||
query += " UNION ALL ";
|
||||
query += fmt::format("SELECT * FROM viewIfPermitted({} ELSE null('{} String'))", select, result_column_name);
|
||||
};
|
||||
|
||||
auto add_column = [&](std::string_view column_name, std::string_view table_name, bool distinct, std::optional<Int64> limit)
|
||||
{
|
||||
add_subquery(
|
||||
fmt::format(
|
||||
"SELECT {}{} FROM system.{}{}",
|
||||
(distinct ? "DISTINCT " : ""),
|
||||
column_name,
|
||||
table_name,
|
||||
(limit ? (" LIMIT " + std::to_string(*limit)) : "")),
|
||||
column_name);
|
||||
};
|
||||
|
||||
add_column("name", "functions", false, {});
|
||||
add_column("name", "table_engines", false, {});
|
||||
add_column("name", "formats", false, {});
|
||||
add_column("name", "table_functions", false, {});
|
||||
add_column("name", "data_type_families", false, {});
|
||||
add_column("name", "merge_tree_settings", false, {});
|
||||
add_column("name", "settings", false, {});
|
||||
|
||||
if (!basic_suggestion)
|
||||
{
|
||||
query << "SELECT cluster FROM system.clusters"
|
||||
" UNION ALL "
|
||||
"SELECT macro FROM system.macros"
|
||||
" UNION ALL "
|
||||
"SELECT policy_name FROM system.storage_policies"
|
||||
" UNION ALL ";
|
||||
add_column("cluster", "clusters", false, {});
|
||||
add_column("macro", "macros", false, {});
|
||||
add_column("policy_name", "storage_policies", false, {});
|
||||
}
|
||||
query << "SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate";
|
||||
|
||||
add_subquery("SELECT concat(func.name, comb.name) AS x FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate", "x");
|
||||
|
||||
/// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero.
|
||||
if (suggestion_limit > 0)
|
||||
{
|
||||
String limit_str = toString(suggestion_limit);
|
||||
query << " UNION ALL "
|
||||
"SELECT name FROM system.databases LIMIT " << limit_str
|
||||
<< " UNION ALL "
|
||||
"SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
|
||||
<< " UNION ALL ";
|
||||
|
||||
add_column("name", "databases", false, suggestion_limit);
|
||||
add_column("name", "tables", true, suggestion_limit);
|
||||
if (!basic_suggestion)
|
||||
{
|
||||
query << "SELECT DISTINCT name FROM system.dictionaries LIMIT " << limit_str
|
||||
<< " UNION ALL ";
|
||||
add_column("name", "dictionaries", true, suggestion_limit);
|
||||
}
|
||||
query << "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
|
||||
add_column("name", "columns", true, suggestion_limit);
|
||||
}
|
||||
query << ") WHERE notEmpty(res)";
|
||||
|
||||
return query.str();
|
||||
query = "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + query + ") WHERE notEmpty(res)";
|
||||
return query;
|
||||
}
|
||||
|
||||
template <typename ConnectionType>
|
||||
|
||||
@ -793,4 +793,18 @@ ColumnPtr makeNullable(const ColumnPtr & column)
|
||||
return ColumnNullable::create(column, ColumnUInt8::create(column->size(), 0));
|
||||
}
|
||||
|
||||
ColumnPtr makeNullableSafe(const ColumnPtr & column)
|
||||
{
|
||||
if (isColumnNullable(*column))
|
||||
return column;
|
||||
|
||||
if (isColumnConst(*column))
|
||||
return ColumnConst::create(makeNullableSafe(assert_cast<const ColumnConst &>(*column).getDataColumnPtr()), column->size());
|
||||
|
||||
if (column->canBeInsideNullable())
|
||||
return makeNullable(column);
|
||||
|
||||
return column;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -223,5 +223,6 @@ private:
|
||||
};
|
||||
|
||||
ColumnPtr makeNullable(const ColumnPtr & column);
|
||||
ColumnPtr makeNullableSafe(const ColumnPtr & column);
|
||||
|
||||
}
|
||||
|
||||
@ -509,7 +509,7 @@ MutableColumnPtr ColumnUnique<ColumnType>::uniqueInsertRangeImpl(
|
||||
if (secondary_index)
|
||||
next_position += secondary_index->size();
|
||||
|
||||
auto insert_key = [&](const StringRef & ref, ReverseIndex<UInt64, ColumnType> & cur_index) -> MutableColumnPtr
|
||||
auto insert_key = [&](StringRef ref, ReverseIndex<UInt64, ColumnType> & cur_index) -> MutableColumnPtr
|
||||
{
|
||||
auto inserted_pos = cur_index.insert(ref);
|
||||
positions[num_added_rows] = inserted_pos;
|
||||
|
||||
@ -92,7 +92,7 @@ struct ReverseIndexHashTableCell
|
||||
|
||||
/// Special case when we want to compare with something not in index_column.
|
||||
/// When we compare something inside column default keyEquals checks only that row numbers are equal.
|
||||
bool keyEquals(const StringRef & object, size_t hash_ [[maybe_unused]], const State & state) const
|
||||
bool keyEquals(StringRef object, size_t hash_ [[maybe_unused]], const State & state) const
|
||||
{
|
||||
auto index = key;
|
||||
if constexpr (has_base_index)
|
||||
@ -322,7 +322,7 @@ public:
|
||||
static constexpr bool is_numeric_column = isNumericColumn(static_cast<ColumnType *>(nullptr));
|
||||
static constexpr bool use_saved_hash = !is_numeric_column;
|
||||
|
||||
UInt64 insert(const StringRef & data);
|
||||
UInt64 insert(StringRef data);
|
||||
|
||||
/// Returns the found data's index in the dictionary. If index is not built, builds it.
|
||||
UInt64 getInsertionPoint(StringRef data)
|
||||
@ -383,7 +383,7 @@ private:
|
||||
|
||||
void buildIndex();
|
||||
|
||||
UInt64 getHash(const StringRef & ref) const
|
||||
UInt64 getHash(StringRef ref) const
|
||||
{
|
||||
if constexpr (is_numeric_column)
|
||||
{
|
||||
@ -478,7 +478,7 @@ ColumnUInt64::MutablePtr ReverseIndex<IndexType, ColumnType>::calcHashes() const
|
||||
}
|
||||
|
||||
template <typename IndexType, typename ColumnType>
|
||||
UInt64 ReverseIndex<IndexType, ColumnType>::insert(const StringRef & data)
|
||||
UInt64 ReverseIndex<IndexType, ColumnType>::insert(StringRef data)
|
||||
{
|
||||
if (!index)
|
||||
buildIndex();
|
||||
|
||||
@ -2,7 +2,6 @@
|
||||
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <base/StringRef.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
@ -76,7 +75,7 @@ public:
|
||||
static void finalizePerformanceCounters();
|
||||
|
||||
/// Returns a non-empty string if the thread is attached to a query
|
||||
static StringRef getQueryId()
|
||||
static std::string_view getQueryId()
|
||||
{
|
||||
if (unlikely(!current_thread))
|
||||
return {};
|
||||
|
||||
63
src/Common/EventRateMeter.h
Normal file
63
src/Common/EventRateMeter.h
Normal file
@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/defines.h>
|
||||
|
||||
#include <Common/ExponentiallySmoothedCounter.h>
|
||||
|
||||
#include <numbers>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Event count measurement with exponential smoothing intended for computing time derivatives
|
||||
class EventRateMeter
|
||||
{
|
||||
public:
|
||||
explicit EventRateMeter(double now, double period_)
|
||||
: period(period_)
|
||||
, half_decay_time(period * std::numbers::ln2) // for `ExponentiallySmoothedAverage::sumWeights()` to be equal to `1/period`
|
||||
{
|
||||
reset(now);
|
||||
}
|
||||
|
||||
/// Add `count` events happened at `now` instant.
|
||||
/// Previous events that are older than `period` from `now` will be forgotten
|
||||
/// in a way to keep average event rate the same, using exponential smoothing.
|
||||
/// NOTE: Adding events into distant past (further than `period`) must be avoided.
|
||||
void add(double now, double count)
|
||||
{
|
||||
if (now - period <= start) // precise counting mode
|
||||
events = ExponentiallySmoothedAverage(events.value + count, now);
|
||||
else // exponential smoothing mode
|
||||
events.add(count, now, half_decay_time);
|
||||
}
|
||||
|
||||
/// Compute average event rate throughout `[now - period, now]` period.
|
||||
/// If measurements are just started (`now - period < start`), then average
|
||||
/// is computed based on shorter `[start; now]` period to avoid initial linear growth.
|
||||
double rate(double now)
|
||||
{
|
||||
add(now, 0);
|
||||
if (unlikely(now <= start))
|
||||
return 0;
|
||||
if (now - period <= start) // precise counting mode
|
||||
return events.value / (now - start);
|
||||
else // exponential smoothing mode
|
||||
return events.get(half_decay_time); // equals to `events.value / period`
|
||||
}
|
||||
|
||||
void reset(double now)
|
||||
{
|
||||
start = now;
|
||||
events = ExponentiallySmoothedAverage();
|
||||
}
|
||||
|
||||
private:
|
||||
const double period;
|
||||
const double half_decay_time;
|
||||
double start; // Instant in past without events before it; when measurement started or reset
|
||||
ExponentiallySmoothedAverage events; // Estimated number of events in the last `period`
|
||||
};
|
||||
|
||||
}
|
||||
@ -104,10 +104,10 @@ String FileSegment::getCallerId()
|
||||
{
|
||||
if (!CurrentThread::isInitialized()
|
||||
|| !CurrentThread::get().getQueryContext()
|
||||
|| CurrentThread::getQueryId().size == 0)
|
||||
|| CurrentThread::getQueryId().empty())
|
||||
return "None:" + toString(getThreadId());
|
||||
|
||||
return CurrentThread::getQueryId().toString() + ":" + toString(getThreadId());
|
||||
return std::string(CurrentThread::getQueryId()) + ":" + toString(getThreadId());
|
||||
}
|
||||
|
||||
String FileSegment::getOrSetDownloader()
|
||||
|
||||
@ -12,7 +12,7 @@ struct StringHashMapCell : public HashMapCell<Key, TMapped, StringHashTableHash,
|
||||
using Base::Base;
|
||||
static constexpr bool need_zero_value_storage = false;
|
||||
// external
|
||||
const StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
|
||||
StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
|
||||
// internal
|
||||
static const Key & getKey(const value_type & value_) { return value_.first; }
|
||||
};
|
||||
@ -32,7 +32,7 @@ struct StringHashMapCell<StringKey16, TMapped> : public HashMapCell<StringKey16,
|
||||
void setZero() { this->value.first.items[1] = 0; }
|
||||
|
||||
// external
|
||||
const StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
|
||||
StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
|
||||
// internal
|
||||
static const StringKey16 & getKey(const value_type & value_) { return value_.first; }
|
||||
};
|
||||
@ -53,7 +53,7 @@ struct StringHashMapCell<StringKey24, TMapped> : public HashMapCell<StringKey24,
|
||||
void setZero() { this->value.first.c = 0; }
|
||||
|
||||
// external
|
||||
const StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
|
||||
StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
|
||||
// internal
|
||||
static const StringKey24 & getKey(const value_type & value_) { return value_.first; }
|
||||
};
|
||||
|
||||
@ -58,7 +58,7 @@ static bool isQueryInitialized()
|
||||
{
|
||||
return CurrentThread::isInitialized()
|
||||
&& CurrentThread::get().getQueryContext()
|
||||
&& CurrentThread::getQueryId().size != 0;
|
||||
&& !CurrentThread::getQueryId().empty();
|
||||
}
|
||||
|
||||
bool IFileCache::isReadOnly()
|
||||
@ -77,7 +77,7 @@ IFileCache::QueryContextPtr IFileCache::getCurrentQueryContext(std::lock_guard<s
|
||||
if (!isQueryInitialized())
|
||||
return nullptr;
|
||||
|
||||
return getQueryContext(CurrentThread::getQueryId().toString(), cache_lock);
|
||||
return getQueryContext(std::string(CurrentThread::getQueryId()), cache_lock);
|
||||
}
|
||||
|
||||
IFileCache::QueryContextPtr IFileCache::getQueryContext(const String & query_id, std::lock_guard<std::mutex> & /* cache_lock */)
|
||||
|
||||
@ -84,7 +84,7 @@ struct DummyJSONParser
|
||||
static Iterator begin() { return {}; }
|
||||
static Iterator end() { return {}; }
|
||||
static size_t size() { return 0; }
|
||||
bool find(const std::string_view &, Element &) const { return false; } /// NOLINT
|
||||
bool find(std::string_view, Element &) const { return false; } /// NOLINT
|
||||
|
||||
#if 0
|
||||
/// Optional: Provides access to an object's element by index.
|
||||
@ -93,7 +93,7 @@ struct DummyJSONParser
|
||||
};
|
||||
|
||||
/// Parses a JSON document, returns the reference to its root element if succeeded.
|
||||
bool parse(const std::string_view &, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
|
||||
bool parse(std::string_view, Element &) { throw Exception{"Functions JSON* are not supported", ErrorCodes::NOT_IMPLEMENTED}; } /// NOLINT
|
||||
|
||||
#if 0
|
||||
/// Optional: Allocates memory to parse JSON documents faster.
|
||||
|
||||
@ -98,7 +98,7 @@ struct RapidJSONParser
|
||||
ALWAYS_INLINE Iterator end() const { return ptr->MemberEnd(); }
|
||||
ALWAYS_INLINE size_t size() const { return ptr->MemberCount(); }
|
||||
|
||||
bool find(const std::string_view & key, Element & result) const
|
||||
bool find(std::string_view key, Element & result) const
|
||||
{
|
||||
auto it = ptr->FindMember(rapidjson::StringRef(key.data(), key.length()));
|
||||
if (it == ptr->MemberEnd())
|
||||
@ -122,7 +122,7 @@ struct RapidJSONParser
|
||||
};
|
||||
|
||||
/// Parses a JSON document, returns the reference to its root element if succeeded.
|
||||
bool parse(const std::string_view & json, Element & result)
|
||||
bool parse(std::string_view json, Element & result)
|
||||
{
|
||||
rapidjson::MemoryStream ms(json.data(), json.size());
|
||||
rapidjson::EncodedInputStream<rapidjson::UTF8<>, rapidjson::MemoryStream> is(ms);
|
||||
|
||||
@ -105,7 +105,7 @@ struct SimdJSONParser
|
||||
ALWAYS_INLINE Iterator end() const { return object.end(); }
|
||||
ALWAYS_INLINE size_t size() const { return object.size(); }
|
||||
|
||||
bool find(const std::string_view & key, Element & result) const
|
||||
bool find(std::string_view key, Element & result) const
|
||||
{
|
||||
auto x = object.at_key(key);
|
||||
if (x.error())
|
||||
@ -131,7 +131,7 @@ struct SimdJSONParser
|
||||
};
|
||||
|
||||
/// Parses a JSON document, returns the reference to its root element if succeeded.
|
||||
bool parse(const std::string_view & json, Element & result)
|
||||
bool parse(std::string_view json, Element & result)
|
||||
{
|
||||
auto document = parser.parse(json.data(), json.size());
|
||||
if (document.error())
|
||||
|
||||
@ -45,7 +45,7 @@ void LRUFileCache::initialize()
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
return;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -841,7 +841,11 @@ void LRUFileCache::loadCacheInfoIntoMemory(std::lock_guard<std::mutex> & cache_l
|
||||
/// cache_base_path / key_prefix / key / offset
|
||||
|
||||
if (!files.empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache already initialized");
|
||||
throw Exception(
|
||||
ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
|
||||
"Cache initialization is partially made. "
|
||||
"This can be a result of a failed first attempt to initialize cache. "
|
||||
"Please, check log for error messages");
|
||||
|
||||
fs::directory_iterator key_prefix_it{cache_base_path};
|
||||
for (; key_prefix_it != fs::directory_iterator(); ++key_prefix_it)
|
||||
|
||||
@ -10,7 +10,7 @@ namespace DB
|
||||
{
|
||||
#pragma GCC diagnostic warning "-Wold-style-cast"
|
||||
|
||||
std::string encodeSHA256(const std::string_view & text)
|
||||
std::string encodeSHA256(std::string_view text)
|
||||
{
|
||||
return encodeSHA256(text.data(), text.size());
|
||||
}
|
||||
@ -21,7 +21,7 @@ std::string encodeSHA256(const void * text, size_t size)
|
||||
encodeSHA256(text, size, reinterpret_cast<unsigned char *>(out.data()));
|
||||
return out;
|
||||
}
|
||||
void encodeSHA256(const std::string_view & text, unsigned char * out)
|
||||
void encodeSHA256(std::string_view text, unsigned char * out)
|
||||
{
|
||||
encodeSHA256(text.data(), text.size(), out);
|
||||
}
|
||||
|
||||
@ -10,10 +10,10 @@ namespace DB
|
||||
{
|
||||
|
||||
/// Encodes `text` and returns it.
|
||||
std::string encodeSHA256(const std::string_view & text);
|
||||
std::string encodeSHA256(std::string_view text);
|
||||
std::string encodeSHA256(const void * text, size_t size);
|
||||
/// `out` must be at least 32 bytes long.
|
||||
void encodeSHA256(const std::string_view & text, unsigned char * out);
|
||||
void encodeSHA256(std::string_view text, unsigned char * out);
|
||||
void encodeSHA256(const void * text, size_t size, unsigned char * out);
|
||||
|
||||
/// Returns concatenation of error strings for all errors that OpenSSL has recorded, emptying the error queue.
|
||||
|
||||
@ -342,6 +342,23 @@ OptimizedRegularExpressionImpl<thread_safe>::OptimizedRegularExpressionImpl(cons
|
||||
}
|
||||
}
|
||||
|
||||
template <bool thread_safe>
|
||||
OptimizedRegularExpressionImpl<thread_safe>::OptimizedRegularExpressionImpl(OptimizedRegularExpressionImpl && rhs) noexcept
|
||||
: is_trivial(rhs.is_trivial)
|
||||
, required_substring_is_prefix(rhs.required_substring_is_prefix)
|
||||
, is_case_insensitive(rhs.is_case_insensitive)
|
||||
, required_substring(std::move(rhs.required_substring))
|
||||
, re2(std::move(rhs.re2))
|
||||
, number_of_subpatterns(rhs.number_of_subpatterns)
|
||||
{
|
||||
if (!required_substring.empty())
|
||||
{
|
||||
if (is_case_insensitive)
|
||||
case_insensitive_substring_searcher.emplace(required_substring.data(), required_substring.size());
|
||||
else
|
||||
case_sensitive_substring_searcher.emplace(required_substring.data(), required_substring.size());
|
||||
}
|
||||
}
|
||||
|
||||
template <bool thread_safe>
|
||||
bool OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject, size_t subject_size) const
|
||||
|
||||
@ -56,6 +56,9 @@ public:
|
||||
using StringPieceType = std::conditional_t<thread_safe, re2::StringPiece, re2_st::StringPiece>;
|
||||
|
||||
OptimizedRegularExpressionImpl(const std::string & regexp_, int options = 0); /// NOLINT
|
||||
/// StringSearcher store pointers to required_substring, it must be updated on move.
|
||||
OptimizedRegularExpressionImpl(OptimizedRegularExpressionImpl && rhs) noexcept;
|
||||
OptimizedRegularExpressionImpl(const OptimizedRegularExpressionImpl & rhs) = delete;
|
||||
|
||||
bool match(const std::string & subject) const
|
||||
{
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
#include "Common/formatReadable.h"
|
||||
#include <Common/TerminalSize.h>
|
||||
#include <Common/UnicodeBar.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include "IO/WriteBufferFromString.h"
|
||||
#include <Databases/DatabaseMemory.h>
|
||||
|
||||
@ -16,16 +17,16 @@ namespace
|
||||
{
|
||||
constexpr UInt64 ALL_THREADS = 0;
|
||||
|
||||
double calculateCPUUsage(DB::ThreadIdToTimeMap times, UInt64 elapsed)
|
||||
UInt64 aggregateCPUUsageNs(DB::ThreadIdToTimeMap times)
|
||||
{
|
||||
auto accumulated = std::accumulate(times.begin(), times.end(), 0,
|
||||
constexpr UInt64 us_to_ns = 1000;
|
||||
return us_to_ns * std::accumulate(times.begin(), times.end(), 0ull,
|
||||
[](UInt64 acc, const auto & elem)
|
||||
{
|
||||
if (elem.first == ALL_THREADS)
|
||||
return acc;
|
||||
return acc + elem.second.time();
|
||||
});
|
||||
return static_cast<double>(accumulated) / elapsed;
|
||||
}
|
||||
}
|
||||
|
||||
@ -55,7 +56,7 @@ void ProgressIndication::resetProgress()
|
||||
write_progress_on_update = false;
|
||||
{
|
||||
std::lock_guard lock(profile_events_mutex);
|
||||
host_cpu_usage.clear();
|
||||
cpu_usage_meter.reset(static_cast<double>(clock_gettime_ns()));
|
||||
thread_data.clear();
|
||||
}
|
||||
}
|
||||
@ -82,15 +83,17 @@ void ProgressIndication::addThreadIdToList(String const & host, UInt64 thread_id
|
||||
thread_to_times[thread_id] = {};
|
||||
}
|
||||
|
||||
void ProgressIndication::updateThreadEventData(HostToThreadTimesMap & new_thread_data, UInt64 elapsed_time)
|
||||
void ProgressIndication::updateThreadEventData(HostToThreadTimesMap & new_thread_data)
|
||||
{
|
||||
std::lock_guard lock(profile_events_mutex);
|
||||
|
||||
UInt64 total_cpu_ns = 0;
|
||||
for (auto & new_host_map : new_thread_data)
|
||||
{
|
||||
host_cpu_usage[new_host_map.first] = calculateCPUUsage(new_host_map.second, elapsed_time);
|
||||
total_cpu_ns += aggregateCPUUsageNs(new_host_map.second);
|
||||
thread_data[new_host_map.first] = std::move(new_host_map.second);
|
||||
}
|
||||
cpu_usage_meter.add(static_cast<double>(clock_gettime_ns()), total_cpu_ns);
|
||||
}
|
||||
|
||||
size_t ProgressIndication::getUsedThreadsCount() const
|
||||
@ -104,14 +107,10 @@ size_t ProgressIndication::getUsedThreadsCount() const
|
||||
});
|
||||
}
|
||||
|
||||
double ProgressIndication::getCPUUsage() const
|
||||
double ProgressIndication::getCPUUsage()
|
||||
{
|
||||
std::lock_guard lock(profile_events_mutex);
|
||||
|
||||
double res = 0;
|
||||
for (const auto & elem : host_cpu_usage)
|
||||
res += elem.second;
|
||||
return res;
|
||||
return cpu_usage_meter.rate(clock_gettime_ns());
|
||||
}
|
||||
|
||||
ProgressIndication::MemoryUsage ProgressIndication::getMemoryUsage() const
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
#include <Interpreters/Context.h>
|
||||
#include <base/types.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
|
||||
#include <Common/EventRateMeter.h>
|
||||
|
||||
/// http://en.wikipedia.org/wiki/ANSI_escape_code
|
||||
#define CLEAR_TO_END_OF_LINE "\033[K"
|
||||
@ -59,12 +59,12 @@ public:
|
||||
|
||||
void addThreadIdToList(String const & host, UInt64 thread_id);
|
||||
|
||||
void updateThreadEventData(HostToThreadTimesMap & new_thread_data, UInt64 elapsed_time);
|
||||
void updateThreadEventData(HostToThreadTimesMap & new_thread_data);
|
||||
|
||||
private:
|
||||
size_t getUsedThreadsCount() const;
|
||||
|
||||
double getCPUUsage() const;
|
||||
double getCPUUsage();
|
||||
|
||||
struct MemoryUsage
|
||||
{
|
||||
@ -91,7 +91,7 @@ private:
|
||||
|
||||
bool write_progress_on_update = false;
|
||||
|
||||
std::unordered_map<String, double> host_cpu_usage;
|
||||
EventRateMeter cpu_usage_meter{static_cast<double>(clock_gettime_ns()), 3'000'000'000 /*ns*/}; // average cpu utilization last 3 second
|
||||
HostToThreadTimesMap thread_data;
|
||||
/// In case of all of the above:
|
||||
/// - clickhouse-local
|
||||
@ -100,7 +100,7 @@ private:
|
||||
///
|
||||
/// It is possible concurrent access to the following:
|
||||
/// - writeProgress() (class properties) (guarded with progress_mutex)
|
||||
/// - thread_data/host_cpu_usage (guarded with profile_events_mutex)
|
||||
/// - thread_data/cpu_usage_meter (guarded with profile_events_mutex)
|
||||
mutable std::mutex profile_events_mutex;
|
||||
mutable std::mutex progress_mutex;
|
||||
};
|
||||
|
||||
@ -4,7 +4,7 @@ namespace DB
|
||||
{
|
||||
namespace
|
||||
{
|
||||
SettingChange * find(SettingsChanges & changes, const std::string_view & name)
|
||||
SettingChange * find(SettingsChanges & changes, std::string_view name)
|
||||
{
|
||||
auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
|
||||
if (it == changes.end())
|
||||
@ -12,7 +12,7 @@ namespace
|
||||
return &*it;
|
||||
}
|
||||
|
||||
const SettingChange * find(const SettingsChanges & changes, const std::string_view & name)
|
||||
const SettingChange * find(const SettingsChanges & changes, std::string_view name)
|
||||
{
|
||||
auto it = std::find_if(changes.begin(), changes.end(), [&name](const SettingChange & change) { return change.name == name; });
|
||||
if (it == changes.end())
|
||||
@ -21,7 +21,7 @@ namespace
|
||||
}
|
||||
}
|
||||
|
||||
bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) const
|
||||
bool SettingsChanges::tryGet(std::string_view name, Field & out_value) const
|
||||
{
|
||||
const auto * change = find(*this, name);
|
||||
if (!change)
|
||||
@ -30,7 +30,7 @@ bool SettingsChanges::tryGet(const std::string_view & name, Field & out_value) c
|
||||
return true;
|
||||
}
|
||||
|
||||
const Field * SettingsChanges::tryGet(const std::string_view & name) const
|
||||
const Field * SettingsChanges::tryGet(std::string_view name) const
|
||||
{
|
||||
const auto * change = find(*this, name);
|
||||
if (!change)
|
||||
@ -38,7 +38,7 @@ const Field * SettingsChanges::tryGet(const std::string_view & name) const
|
||||
return &change->value;
|
||||
}
|
||||
|
||||
Field * SettingsChanges::tryGet(const std::string_view & name)
|
||||
Field * SettingsChanges::tryGet(std::string_view name)
|
||||
{
|
||||
auto * change = find(*this, name);
|
||||
if (!change)
|
||||
|
||||
@ -14,8 +14,8 @@ struct SettingChange
|
||||
Field value;
|
||||
|
||||
SettingChange() = default;
|
||||
SettingChange(const std::string_view & name_, const Field & value_) : name(name_), value(value_) {}
|
||||
SettingChange(const std::string_view & name_, Field && value_) : name(name_), value(std::move(value_)) {}
|
||||
SettingChange(std::string_view name_, const Field & value_) : name(name_), value(value_) {}
|
||||
SettingChange(std::string_view name_, Field && value_) : name(name_), value(std::move(value_)) {}
|
||||
|
||||
friend bool operator ==(const SettingChange & lhs, const SettingChange & rhs) { return (lhs.name == rhs.name) && (lhs.value == rhs.value); }
|
||||
friend bool operator !=(const SettingChange & lhs, const SettingChange & rhs) { return !(lhs == rhs); }
|
||||
@ -27,9 +27,9 @@ class SettingsChanges : public std::vector<SettingChange>
|
||||
public:
|
||||
using std::vector<SettingChange>::vector;
|
||||
|
||||
bool tryGet(const std::string_view & name, Field & out_value) const;
|
||||
const Field * tryGet(const std::string_view & name) const;
|
||||
Field * tryGet(const std::string_view & name);
|
||||
bool tryGet(std::string_view name, Field & out_value) const;
|
||||
const Field * tryGet(std::string_view name) const;
|
||||
Field * tryGet(std::string_view name);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -1,9 +1,7 @@
|
||||
#include <sys/types.h>
|
||||
#include <sys/wait.h>
|
||||
#include <fcntl.h>
|
||||
#include <dlfcn.h>
|
||||
#include <unistd.h>
|
||||
#include <ctime>
|
||||
#include <csignal>
|
||||
|
||||
#include <Common/logger_useful.h>
|
||||
@ -13,6 +11,7 @@
|
||||
#include <Common/PipeFDs.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <Common/waitForPid.h>
|
||||
|
||||
|
||||
namespace
|
||||
@ -94,53 +93,15 @@ ShellCommand::~ShellCommand()
|
||||
|
||||
bool ShellCommand::tryWaitProcessWithTimeout(size_t timeout_in_seconds)
|
||||
{
|
||||
int status = 0;
|
||||
|
||||
LOG_TRACE(getLogger(), "Try wait for shell command pid {} with timeout {}", pid, timeout_in_seconds);
|
||||
|
||||
wait_called = true;
|
||||
struct timespec interval {.tv_sec = 1, .tv_nsec = 0};
|
||||
|
||||
in.close();
|
||||
out.close();
|
||||
err.close();
|
||||
|
||||
if (timeout_in_seconds == 0)
|
||||
{
|
||||
/// If there is no timeout before signal try to waitpid 1 time without block so we can avoid sending
|
||||
/// signal if process is already normally terminated.
|
||||
|
||||
int waitpid_res = waitpid(pid, &status, WNOHANG);
|
||||
bool process_terminated_normally = (waitpid_res == pid);
|
||||
return process_terminated_normally;
|
||||
}
|
||||
|
||||
/// If timeout is positive try waitpid without block in loop until
|
||||
/// process is normally terminated or waitpid return error
|
||||
|
||||
while (timeout_in_seconds != 0)
|
||||
{
|
||||
int waitpid_res = waitpid(pid, &status, WNOHANG);
|
||||
bool process_terminated_normally = (waitpid_res == pid);
|
||||
|
||||
if (process_terminated_normally)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
else if (waitpid_res == 0)
|
||||
{
|
||||
--timeout_in_seconds;
|
||||
nanosleep(&interval, nullptr);
|
||||
|
||||
continue;
|
||||
}
|
||||
else if (waitpid_res == -1 && errno != EINTR)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
return waitForPid(pid, timeout_in_seconds);
|
||||
}
|
||||
|
||||
void ShellCommand::logCommand(const char * filename, char * const argv[])
|
||||
|
||||
@ -49,12 +49,12 @@ struct SpaceSavingArena
|
||||
template <>
|
||||
struct SpaceSavingArena<StringRef>
|
||||
{
|
||||
StringRef emplace(const StringRef & key)
|
||||
StringRef emplace(StringRef key)
|
||||
{
|
||||
return copyStringInArena(arena, key);
|
||||
}
|
||||
|
||||
void free(const StringRef & key)
|
||||
void free(StringRef key)
|
||||
{
|
||||
if (key.data)
|
||||
arena.free(const_cast<char *>(key.data), key.size);
|
||||
|
||||
@ -147,7 +147,7 @@ inline bool isPunctuationASCII(char c)
|
||||
}
|
||||
|
||||
|
||||
inline bool isValidIdentifier(const std::string_view & str)
|
||||
inline bool isValidIdentifier(std::string_view str)
|
||||
{
|
||||
return !str.empty()
|
||||
&& isValidIdentifierBegin(str[0])
|
||||
|
||||
@ -20,13 +20,13 @@ TLDList::TLDList(size_t size)
|
||||
: tld_container(size)
|
||||
, pool(std::make_unique<Arena>(10 << 20))
|
||||
{}
|
||||
bool TLDList::insert(const StringRef & host)
|
||||
bool TLDList::insert(StringRef host)
|
||||
{
|
||||
bool inserted;
|
||||
tld_container.emplace(DB::ArenaKeyHolder{host, *pool}, inserted);
|
||||
return inserted;
|
||||
}
|
||||
bool TLDList::has(const StringRef & host) const
|
||||
bool TLDList::has(StringRef host) const
|
||||
{
|
||||
return tld_container.has(host);
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user