mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 17:41:59 +00:00
Merge branch 'master' into better-support-gcp
This commit is contained in:
alesapin
GitHub
commit
0f8eed98c4
2
.github/workflows/backport.yml
vendored
2
.github/workflows/backport.yml
vendored
@ -13,7 +13,7 @@ on: # yamllint disable-line rule:truthy
|
||||
|
||||
jobs:
|
||||
CherryPick:
|
||||
runs-on: [self-hosted, style-checker]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Set envs
|
||||
# https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
#include <replxx.hxx>
|
||||
|
||||
#include <base/types.h>
|
||||
#include <base/defines.h>
|
||||
|
||||
class LineReader
|
||||
{
|
||||
@ -20,8 +21,8 @@ public:
|
||||
void addWords(Words && new_words);
|
||||
|
||||
private:
|
||||
Words words;
|
||||
Words words_no_case;
|
||||
Words words TSA_GUARDED_BY(mutex);
|
||||
Words words_no_case TSA_GUARDED_BY(mutex);
|
||||
|
||||
std::mutex mutex;
|
||||
};
|
||||
@ -29,7 +30,7 @@ public:
|
||||
using Patterns = std::vector<const char *>;
|
||||
|
||||
LineReader(const String & history_file_path, bool multiline, Patterns extenders, Patterns delimiters);
|
||||
virtual ~LineReader() {}
|
||||
virtual ~LineReader() = default;
|
||||
|
||||
/// Reads the whole line until delimiter (in multiline mode) or until the last line without extender.
|
||||
/// If resulting line is empty, it means the user interrupted the input.
|
||||
|
||||
@ -124,6 +124,23 @@
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Macros for Clang Thread Safety Analysis (TSA). They can be safely ignored by other compilers.
|
||||
// Feel free to extend, but please stay close to https://clang.llvm.org/docs/ThreadSafetyAnalysis.html#mutexheader
|
||||
#if defined(__clang__)
|
||||
# define TSA_GUARDED_BY(...) __attribute__((guarded_by(__VA_ARGS__))) // data is protected by given capability
|
||||
# define TSA_PT_GUARDED_BY(...) __attribute__((pt_guarded_by(__VA_ARGS__))) // pointed-to data is protected by the given capability
|
||||
# define TSA_REQUIRES(...) __attribute__((requires_capability(__VA_ARGS__))) // thread needs exclusive possession of given capability
|
||||
# define TSA_REQUIRES_SHARED(...) __attribute__((requires_shared_capability(__VA_ARGS__))) // thread needs shared possession of given capability
|
||||
# define TSA_ACQUIRED_AFTER(...) __attribute__((acquired_after(__VA_ARGS__))) // annotated lock must be locked after given lock
|
||||
# define TSA_NO_THREAD_SAFETY_ANALYSIS __attribute__((no_thread_safety_analysis)) // disable TSA for a function
|
||||
#else
|
||||
# define TSA_GUARDED_BY(...)
|
||||
# define TSA_PT_GUARDED_BY(...)
|
||||
# define TSA_REQUIRES(...)
|
||||
# define TSA_REQUIRES_SHARED(...)
|
||||
# define TSA_NO_THREAD_SAFETY_ANALYSIS
|
||||
#endif
|
||||
|
||||
/// A template function for suppressing warnings about unused variables or function results.
|
||||
template <typename... Args>
|
||||
constexpr void UNUSED(Args &&... args [[maybe_unused]])
|
||||
|
||||
@ -260,4 +260,35 @@ TRAP(mq_timedreceive)
|
||||
TRAP(wordexp)
|
||||
TRAP(wordfree)
|
||||
|
||||
/// C11 threading primitives are not supported by ThreadSanitizer.
|
||||
/// Also we should avoid using them for compatibility with old libc.
|
||||
TRAP(thrd_create)
|
||||
TRAP(thrd_equal)
|
||||
TRAP(thrd_current)
|
||||
TRAP(thrd_sleep)
|
||||
TRAP(thrd_yield)
|
||||
TRAP(thrd_exit)
|
||||
TRAP(thrd_detach)
|
||||
TRAP(thrd_join)
|
||||
|
||||
TRAP(mtx_init)
|
||||
TRAP(mtx_lock)
|
||||
TRAP(mtx_timedlock)
|
||||
TRAP(mtx_trylock)
|
||||
TRAP(mtx_unlock)
|
||||
TRAP(mtx_destroy)
|
||||
TRAP(call_once)
|
||||
|
||||
TRAP(cnd_init)
|
||||
TRAP(cnd_signal)
|
||||
TRAP(cnd_broadcast)
|
||||
TRAP(cnd_wait)
|
||||
TRAP(cnd_timedwait)
|
||||
TRAP(cnd_destroy)
|
||||
|
||||
TRAP(tss_create)
|
||||
TRAP(tss_get)
|
||||
TRAP(tss_set)
|
||||
TRAP(tss_delete)
|
||||
|
||||
#endif
|
||||
|
||||
@ -112,7 +112,7 @@ endif()
|
||||
# Archiver
|
||||
|
||||
if (COMPILER_GCC)
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11")
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-14" "llvm-ar-13" "llvm-ar-12")
|
||||
else ()
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar-${COMPILER_VERSION_MAJOR}" "llvm-ar")
|
||||
endif ()
|
||||
@ -126,7 +126,7 @@ message(STATUS "Using archiver: ${CMAKE_AR}")
|
||||
# Ranlib
|
||||
|
||||
if (COMPILER_GCC)
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11")
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-14" "llvm-ranlib-13" "llvm-ranlib-12")
|
||||
else ()
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib-${COMPILER_VERSION_MAJOR}" "llvm-ranlib")
|
||||
endif ()
|
||||
@ -140,7 +140,7 @@ message(STATUS "Using ranlib: ${CMAKE_RANLIB}")
|
||||
# Install Name Tool
|
||||
|
||||
if (COMPILER_GCC)
|
||||
find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool" "llvm-install-name-tool-13" "llvm-install-name-tool-12" "llvm-install-name-tool-11")
|
||||
find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool" "llvm-install-name-tool-14" "llvm-install-name-tool-13" "llvm-install-name-tool-12")
|
||||
else ()
|
||||
find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool-${COMPILER_VERSION_MAJOR}" "llvm-install-name-tool")
|
||||
endif ()
|
||||
@ -154,7 +154,7 @@ message(STATUS "Using install-name-tool: ${CMAKE_INSTALL_NAME_TOOL}")
|
||||
# Objcopy
|
||||
|
||||
if (COMPILER_GCC)
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-13" "llvm-objcopy-12" "llvm-objcopy-11" "objcopy")
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-14" "llvm-objcopy-13" "llvm-objcopy-12" "objcopy")
|
||||
else ()
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy-${COMPILER_VERSION_MAJOR}" "llvm-objcopy" "objcopy")
|
||||
endif ()
|
||||
@ -168,7 +168,7 @@ endif ()
|
||||
# Strip
|
||||
|
||||
if (COMPILER_GCC)
|
||||
find_program (STRIP_PATH NAMES "llvm-strip" "llvm-strip-13" "llvm-strip-12" "llvm-strip-11" "strip")
|
||||
find_program (STRIP_PATH NAMES "llvm-strip" "llvm-strip-14" "llvm-strip-13" "llvm-strip-12" "strip")
|
||||
else ()
|
||||
find_program (STRIP_PATH NAMES "llvm-strip-${COMPILER_VERSION_MAJOR}" "llvm-strip" "strip")
|
||||
endif ()
|
||||
|
||||
@ -19,7 +19,6 @@ if (COMPILER_CLANG)
|
||||
# Add some warnings that are not available even with -Wall -Wextra -Wpedantic.
|
||||
# We want to get everything out of the compiler for code quality.
|
||||
add_warning(everything)
|
||||
|
||||
add_warning(pedantic)
|
||||
no_warning(vla-extension)
|
||||
no_warning(zero-length-array)
|
||||
@ -51,6 +50,7 @@ if (COMPILER_CLANG)
|
||||
no_warning(vla)
|
||||
no_warning(weak-template-vtables)
|
||||
no_warning(weak-vtables)
|
||||
no_warning(thread-safety-negative) # experimental flag, too many false positives
|
||||
# TODO Enable conversion, sign-conversion, double-promotion warnings.
|
||||
elseif (COMPILER_GCC)
|
||||
# Add compiler options only to c++ compiler
|
||||
|
||||
2
contrib/curl
vendored
2
contrib/curl
vendored
@ -1 +1 @@
|
||||
Subproject commit 801bd5138ce31aa0d906fa4e2eabfc599d74e793
|
||||
Subproject commit 462196e6b4a47f924293a0e26b8e9c23d37ac26f
|
||||
@ -84,7 +84,6 @@ set (SRCS
|
||||
"${LIBRARY_DIR}/lib/gopher.c"
|
||||
"${LIBRARY_DIR}/lib/idn_win32.c"
|
||||
"${LIBRARY_DIR}/lib/http_proxy.c"
|
||||
"${LIBRARY_DIR}/lib/non-ascii.c"
|
||||
"${LIBRARY_DIR}/lib/asyn-thread.c"
|
||||
"${LIBRARY_DIR}/lib/curl_gssapi.c"
|
||||
"${LIBRARY_DIR}/lib/http_ntlm.c"
|
||||
@ -93,10 +92,8 @@ set (SRCS
|
||||
"${LIBRARY_DIR}/lib/curl_sasl.c"
|
||||
"${LIBRARY_DIR}/lib/rand.c"
|
||||
"${LIBRARY_DIR}/lib/curl_multibyte.c"
|
||||
"${LIBRARY_DIR}/lib/hostcheck.c"
|
||||
"${LIBRARY_DIR}/lib/conncache.c"
|
||||
"${LIBRARY_DIR}/lib/dotdot.c"
|
||||
"${LIBRARY_DIR}/lib/x509asn1.c"
|
||||
"${LIBRARY_DIR}/lib/http2.c"
|
||||
"${LIBRARY_DIR}/lib/smb.c"
|
||||
"${LIBRARY_DIR}/lib/curl_endian.c"
|
||||
@ -120,6 +117,9 @@ set (SRCS
|
||||
"${LIBRARY_DIR}/lib/http_aws_sigv4.c"
|
||||
"${LIBRARY_DIR}/lib/mqtt.c"
|
||||
"${LIBRARY_DIR}/lib/rename.c"
|
||||
"${LIBRARY_DIR}/lib/h2h3.c"
|
||||
"${LIBRARY_DIR}/lib/headers.c"
|
||||
"${LIBRARY_DIR}/lib/timediff.c"
|
||||
"${LIBRARY_DIR}/lib/vauth/vauth.c"
|
||||
"${LIBRARY_DIR}/lib/vauth/cleartext.c"
|
||||
"${LIBRARY_DIR}/lib/vauth/cram.c"
|
||||
@ -142,11 +142,13 @@ set (SRCS
|
||||
"${LIBRARY_DIR}/lib/vtls/sectransp.c"
|
||||
"${LIBRARY_DIR}/lib/vtls/gskit.c"
|
||||
"${LIBRARY_DIR}/lib/vtls/mbedtls.c"
|
||||
"${LIBRARY_DIR}/lib/vtls/mesalink.c"
|
||||
"${LIBRARY_DIR}/lib/vtls/bearssl.c"
|
||||
"${LIBRARY_DIR}/lib/vtls/keylog.c"
|
||||
"${LIBRARY_DIR}/lib/vtls/x509asn1.c"
|
||||
"${LIBRARY_DIR}/lib/vtls/hostcheck.c"
|
||||
"${LIBRARY_DIR}/lib/vquic/ngtcp2.c"
|
||||
"${LIBRARY_DIR}/lib/vquic/quiche.c"
|
||||
"${LIBRARY_DIR}/lib/vquic/msh3.c"
|
||||
"${LIBRARY_DIR}/lib/vssh/libssh2.c"
|
||||
"${LIBRARY_DIR}/lib/vssh/libssh.c"
|
||||
)
|
||||
|
||||
@ -78,6 +78,9 @@ target_compile_options(cxx PUBLIC $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++>)
|
||||
# Third party library may have substandard code.
|
||||
target_compile_options(cxx PRIVATE -w)
|
||||
|
||||
# Enable support for Clang-Thread-Safety-Analysis in libcxx
|
||||
target_compile_definitions(cxx PUBLIC -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS)
|
||||
|
||||
target_link_libraries(cxx PUBLIC cxxabi)
|
||||
|
||||
# For __udivmodti4, __divmodti4.
|
||||
|
||||
2
contrib/librdkafka
vendored
2
contrib/librdkafka
vendored
@ -1 +1 @@
|
||||
Subproject commit b8554f1682062c85ba519eb54ef2f90e02b812cb
|
||||
Subproject commit 6062e711a919fb3b669b243b7dceabd045d0e4a2
|
||||
@ -37,38 +37,13 @@ export FASTTEST_DATA
|
||||
export FASTTEST_OUT
|
||||
export PATH
|
||||
|
||||
server_pid=none
|
||||
|
||||
function stop_server
|
||||
{
|
||||
if ! kill -0 -- "$server_pid"
|
||||
then
|
||||
echo "ClickHouse server pid '$server_pid' is not running"
|
||||
return 0
|
||||
fi
|
||||
|
||||
for _ in {1..60}
|
||||
do
|
||||
if ! pkill -f "clickhouse-server" && ! kill -- "$server_pid" ; then break ; fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if kill -0 -- "$server_pid"
|
||||
then
|
||||
pstree -apgT
|
||||
jobs
|
||||
echo "Failed to kill the ClickHouse server pid '$server_pid'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
server_pid=none
|
||||
}
|
||||
|
||||
function start_server
|
||||
{
|
||||
set -m # Spawn server in its own process groups
|
||||
|
||||
local opts=(
|
||||
--config-file "$FASTTEST_DATA/config.xml"
|
||||
--pid-file "$FASTTEST_DATA/clickhouse-server.pid"
|
||||
--
|
||||
--path "$FASTTEST_DATA"
|
||||
--user_files_path "$FASTTEST_DATA/user_files"
|
||||
@ -76,40 +51,22 @@ function start_server
|
||||
--keeper_server.storage_path "$FASTTEST_DATA/coordination"
|
||||
)
|
||||
clickhouse-server "${opts[@]}" &>> "$FASTTEST_OUTPUT/server.log" &
|
||||
server_pid=$!
|
||||
set +m
|
||||
|
||||
if [ "$server_pid" == "0" ]
|
||||
then
|
||||
echo "Failed to start ClickHouse server"
|
||||
# Avoid zero PID because `kill` treats it as our process group PID.
|
||||
server_pid="none"
|
||||
return 1
|
||||
fi
|
||||
|
||||
for _ in {1..60}
|
||||
do
|
||||
if clickhouse-client --query "select 1" || ! kill -0 -- "$server_pid"
|
||||
then
|
||||
for _ in {1..60}; do
|
||||
if clickhouse-client --query "select 1"; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if ! clickhouse-client --query "select 1"
|
||||
then
|
||||
if ! clickhouse-client --query "select 1"; then
|
||||
echo "Failed to wait until ClickHouse server starts."
|
||||
server_pid="none"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! kill -0 -- "$server_pid"
|
||||
then
|
||||
echo "Wrong clickhouse server started: PID '$server_pid' we started is not running, but '$(pgrep -f clickhouse-server)' is running"
|
||||
server_pid="none"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local server_pid
|
||||
server_pid="$(cat "$FASTTEST_DATA/clickhouse-server.pid")"
|
||||
echo "ClickHouse server pid '$server_pid' started and responded"
|
||||
}
|
||||
|
||||
@ -254,9 +211,6 @@ function run_tests
|
||||
clickhouse-server --version
|
||||
clickhouse-test --help
|
||||
|
||||
# Kill the server in case we are running locally and not in docker
|
||||
stop_server ||:
|
||||
|
||||
start_server
|
||||
|
||||
set +e
|
||||
@ -284,6 +238,8 @@ function run_tests
|
||||
| ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee "$FASTTEST_OUTPUT/test_result.txt"
|
||||
set -e
|
||||
|
||||
clickhouse stop --pid-path "$FASTTEST_DATA"
|
||||
}
|
||||
|
||||
case "$stage" in
|
||||
|
||||
@ -125,16 +125,7 @@ function filter_exists_and_template
|
||||
function stop_server
|
||||
{
|
||||
clickhouse-client --query "select elapsed, query from system.processes" ||:
|
||||
killall clickhouse-server ||:
|
||||
for _ in {1..10}
|
||||
do
|
||||
if ! pgrep -f clickhouse-server
|
||||
then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
killall -9 clickhouse-server ||:
|
||||
clickhouse stop
|
||||
|
||||
# Debug.
|
||||
date
|
||||
@ -159,10 +150,12 @@ function fuzz
|
||||
NEW_TESTS_OPT="${NEW_TESTS_OPT:-}"
|
||||
fi
|
||||
|
||||
mkdir -p /var/run/clickhouse-server
|
||||
|
||||
# interferes with gdb
|
||||
export CLICKHOUSE_WATCHDOG_ENABLE=0
|
||||
# NOTE: we use process substitution here to preserve keep $! as a pid of clickhouse-server
|
||||
clickhouse-server --config-file db/config.xml -- --path db > >(tail -100000 > server.log) 2>&1 &
|
||||
clickhouse-server --config-file db/config.xml --pid-file /var/run/clickhouse-server/clickhouse-server.pid -- --path db > >(tail -100000 > server.log) 2>&1 &
|
||||
server_pid=$!
|
||||
|
||||
kill -0 $server_pid
|
||||
|
||||
@ -21,7 +21,7 @@ export NUM_QUERIES=1000
|
||||
( java -jar target/sqlancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES --username default --password "" clickhouse --oracle TLPDistinct | tee /test_output/TLPDistinct.out ) 3>&1 1>&2 2>&3 | tee /test_output/TLPDistinct.err
|
||||
( java -jar target/sqlancer-*.jar --num-threads 10 --timeout-seconds $TIMEOUT --num-queries $NUM_QUERIES --username default --password "" clickhouse --oracle TLPAggregate | tee /test_output/TLPAggregate.out ) 3>&1 1>&2 2>&3 | tee /test_output/TLPAggregate.err
|
||||
|
||||
service clickhouse-server stop && sleep 10
|
||||
service clickhouse stop
|
||||
|
||||
ls /var/log/clickhouse-server/
|
||||
tar czf /test_output/logs.tar.gz -C /var/log/clickhouse-server/ .
|
||||
|
||||
@ -22,17 +22,23 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
|
||||
function start()
|
||||
{
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
mkdir -p /var/run/clickhouse-server1
|
||||
sudo chown clickhouse:clickhouse /var/run/clickhouse-server1
|
||||
# NOTE We run "clickhouse server" instead of "clickhouse-server"
|
||||
# to make "pidof clickhouse-server" return single pid of the main instance.
|
||||
# We wil run main instance using "service clickhouse-server start"
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \
|
||||
--pid-file /var/run/clickhouse-server1/clickhouse-server.pid \
|
||||
-- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \
|
||||
--logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \
|
||||
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
|
||||
--mysql_port 19004 --postgresql_port 19005 \
|
||||
--keeper_server.tcp_port 19181 --keeper_server.server_id 2
|
||||
|
||||
mkdir -p /var/run/clickhouse-server2
|
||||
sudo chown clickhouse:clickhouse /var/run/clickhouse-server2
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server2/config.xml --daemon \
|
||||
--pid-file /var/run/clickhouse-server2/clickhouse-server.pid \
|
||||
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
|
||||
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
|
||||
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
|
||||
@ -135,6 +141,12 @@ ls -la /
|
||||
|
||||
/process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
|
||||
sudo clickhouse stop ||:
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
|
||||
sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||:
|
||||
fi
|
||||
|
||||
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
|
||||
|
||||
pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||:
|
||||
|
||||
@ -41,15 +41,18 @@ if [ "$NUM_TRIES" -gt "1" ]; then
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
|
||||
|
||||
mkdir -p /var/run/clickhouse-server
|
||||
# simpliest way to forward env variables to server
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon --pid-file /var/run/clickhouse-server/clickhouse-server.pid
|
||||
else
|
||||
sudo clickhouse start
|
||||
fi
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
|
||||
mkdir -p /var/run/clickhouse-server1
|
||||
sudo chown clickhouse:clickhouse /var/run/clickhouse-server1
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \
|
||||
--pid-file /var/run/clickhouse-server1/clickhouse-server.pid \
|
||||
-- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \
|
||||
--logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \
|
||||
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
|
||||
@ -57,7 +60,10 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
|
||||
--keeper_server.tcp_port 19181 --keeper_server.server_id 2 \
|
||||
--macros.replica r2 # It doesn't work :(
|
||||
|
||||
mkdir -p /var/run/clickhouse-server2
|
||||
sudo chown clickhouse:clickhouse /var/run/clickhouse-server2
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server2/config.xml --daemon \
|
||||
--pid-file /var/run/clickhouse-server2/clickhouse-server.pid \
|
||||
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
|
||||
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
|
||||
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
|
||||
@ -133,18 +139,10 @@ clickhouse-client -q "system flush logs" ||:
|
||||
# Stop server so we can safely read data with clickhouse-local.
|
||||
# Why do we read data with clickhouse-local?
|
||||
# Because it's the simplest way to read it when server has crashed.
|
||||
if [ "$NUM_TRIES" -gt "1" ]; then
|
||||
clickhouse-client -q "system shutdown" ||:
|
||||
sleep 10
|
||||
else
|
||||
sudo clickhouse stop ||:
|
||||
fi
|
||||
|
||||
|
||||
sudo clickhouse stop ||:
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
clickhouse-client --port 19000 -q "system shutdown" ||:
|
||||
clickhouse-client --port 29000 -q "system shutdown" ||:
|
||||
sleep 10
|
||||
sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
|
||||
sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||:
|
||||
fi
|
||||
|
||||
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
|
||||
|
||||
@ -7,26 +7,29 @@ set -x
|
||||
|
||||
# Thread Fuzzer allows to check more permutations of possible thread scheduling
|
||||
# and find more potential issues.
|
||||
#
|
||||
# But under thread fuzzer, TSan build is too slow and this produces some flaky
|
||||
# tests, so for now, as a temporary solution it had been disabled.
|
||||
if ! test -f package_folder/clickhouse-server*tsan*.deb; then
|
||||
export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
|
||||
export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
|
||||
export THREAD_FUZZER_SLEEP_TIME_US=100000
|
||||
|
||||
export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
|
||||
export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
|
||||
export THREAD_FUZZER_SLEEP_TIME_US=100000
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY=1
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_MIGRATE_PROBABILITY=1
|
||||
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY=1
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_MIGRATE_PROBABILITY=1
|
||||
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
|
||||
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
|
||||
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
|
||||
fi
|
||||
|
||||
function install_packages()
|
||||
{
|
||||
|
||||
@ -18,8 +18,10 @@ def get_options(i, backward_compatibility_check):
|
||||
options.append("--db-engine=Ordinary")
|
||||
|
||||
if i % 3 == 2 and not backward_compatibility_check:
|
||||
options.append('''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i))
|
||||
client_options.append('allow_experimental_database_replicated=1')
|
||||
options.append(
|
||||
'''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i)
|
||||
)
|
||||
client_options.append("allow_experimental_database_replicated=1")
|
||||
|
||||
# If database name is not specified, new database is created for each functional test.
|
||||
# Run some threads with one database for all tests.
|
||||
@ -37,38 +39,58 @@ def get_options(i, backward_compatibility_check):
|
||||
|
||||
if i % 15 == 11:
|
||||
client_options.append("join_algorithm='auto'")
|
||||
client_options.append('max_rows_in_join=1000')
|
||||
client_options.append("max_rows_in_join=1000")
|
||||
|
||||
if i == 13:
|
||||
client_options.append('memory_tracker_fault_probability=0.001')
|
||||
client_options.append("memory_tracker_fault_probability=0.001")
|
||||
|
||||
if client_options:
|
||||
options.append(" --client-option " + ' '.join(client_options))
|
||||
options.append(" --client-option " + " ".join(client_options))
|
||||
|
||||
return ' '.join(options)
|
||||
return " ".join(options)
|
||||
|
||||
|
||||
def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_time_limit, backward_compatibility_check):
|
||||
backward_compatibility_check_option = '--backward-compatibility-check' if backward_compatibility_check else ''
|
||||
global_time_limit_option = ''
|
||||
def run_func_test(
|
||||
cmd,
|
||||
output_prefix,
|
||||
num_processes,
|
||||
skip_tests_option,
|
||||
global_time_limit,
|
||||
backward_compatibility_check,
|
||||
):
|
||||
backward_compatibility_check_option = (
|
||||
"--backward-compatibility-check" if backward_compatibility_check else ""
|
||||
)
|
||||
global_time_limit_option = ""
|
||||
if global_time_limit:
|
||||
global_time_limit_option = "--global_time_limit={}".format(global_time_limit)
|
||||
|
||||
output_paths = [os.path.join(output_prefix, "stress_test_run_{}.txt".format(i)) for i in range(num_processes)]
|
||||
output_paths = [
|
||||
os.path.join(output_prefix, "stress_test_run_{}.txt".format(i))
|
||||
for i in range(num_processes)
|
||||
]
|
||||
pipes = []
|
||||
for i in range(0, len(output_paths)):
|
||||
f = open(output_paths[i], 'w')
|
||||
full_command = "{} {} {} {} {}".format(cmd, get_options(i, backward_compatibility_check), global_time_limit_option, skip_tests_option, backward_compatibility_check_option)
|
||||
f = open(output_paths[i], "w")
|
||||
full_command = "{} {} {} {} {}".format(
|
||||
cmd,
|
||||
get_options(i, backward_compatibility_check),
|
||||
global_time_limit_option,
|
||||
skip_tests_option,
|
||||
backward_compatibility_check_option,
|
||||
)
|
||||
logging.info("Run func tests '%s'", full_command)
|
||||
p = Popen(full_command, shell=True, stdout=f, stderr=f)
|
||||
pipes.append(p)
|
||||
time.sleep(0.5)
|
||||
return pipes
|
||||
|
||||
|
||||
def compress_stress_logs(output_path, files_prefix):
|
||||
cmd = f"cd {output_path} && tar -zcf stress_run_logs.tar.gz {files_prefix}* && rm {files_prefix}*"
|
||||
check_output(cmd, shell=True)
|
||||
|
||||
|
||||
def call_with_retry(query, timeout=30, retry_count=5):
|
||||
for i in range(retry_count):
|
||||
code = call(query, shell=True, stderr=STDOUT, timeout=timeout)
|
||||
@ -77,6 +99,7 @@ def call_with_retry(query, timeout=30, retry_count=5):
|
||||
else:
|
||||
break
|
||||
|
||||
|
||||
def make_query_command(query):
|
||||
return f"""clickhouse client -q "{query}" --max_untracked_memory=1Gi --memory_profiler_step=1Gi --max_memory_usage_for_user=0"""
|
||||
|
||||
@ -93,28 +116,34 @@ def prepare_for_hung_check(drop_databases):
|
||||
# ThreadFuzzer significantly slows down server and causes false-positive hung check failures
|
||||
call_with_retry("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'")
|
||||
|
||||
call_with_retry(make_query_command('SELECT 1 FORMAT Null'))
|
||||
call_with_retry(make_query_command("SELECT 1 FORMAT Null"))
|
||||
|
||||
# Some tests execute SYSTEM STOP MERGES or similar queries.
|
||||
# It may cause some ALTERs to hang.
|
||||
# Possibly we should fix tests and forbid to use such queries without specifying table.
|
||||
call_with_retry(make_query_command('SYSTEM START MERGES'))
|
||||
call_with_retry(make_query_command('SYSTEM START DISTRIBUTED SENDS'))
|
||||
call_with_retry(make_query_command('SYSTEM START TTL MERGES'))
|
||||
call_with_retry(make_query_command('SYSTEM START MOVES'))
|
||||
call_with_retry(make_query_command('SYSTEM START FETCHES'))
|
||||
call_with_retry(make_query_command('SYSTEM START REPLICATED SENDS'))
|
||||
call_with_retry(make_query_command('SYSTEM START REPLICATION QUEUES'))
|
||||
call_with_retry(make_query_command('SYSTEM DROP MARK CACHE'))
|
||||
call_with_retry(make_query_command("SYSTEM START MERGES"))
|
||||
call_with_retry(make_query_command("SYSTEM START DISTRIBUTED SENDS"))
|
||||
call_with_retry(make_query_command("SYSTEM START TTL MERGES"))
|
||||
call_with_retry(make_query_command("SYSTEM START MOVES"))
|
||||
call_with_retry(make_query_command("SYSTEM START FETCHES"))
|
||||
call_with_retry(make_query_command("SYSTEM START REPLICATED SENDS"))
|
||||
call_with_retry(make_query_command("SYSTEM START REPLICATION QUEUES"))
|
||||
call_with_retry(make_query_command("SYSTEM DROP MARK CACHE"))
|
||||
|
||||
# Issue #21004, live views are experimental, so let's just suppress it
|
||||
call_with_retry(make_query_command("KILL QUERY WHERE upper(query) LIKE 'WATCH %'"))
|
||||
|
||||
# Kill other queries which known to be slow
|
||||
# It's query from 01232_preparing_sets_race_condition_long, it may take up to 1000 seconds in slow builds
|
||||
call_with_retry(make_query_command("KILL QUERY WHERE query LIKE 'insert into tableB select %'"))
|
||||
call_with_retry(
|
||||
make_query_command("KILL QUERY WHERE query LIKE 'insert into tableB select %'")
|
||||
)
|
||||
# Long query from 00084_external_agregation
|
||||
call_with_retry(make_query_command("KILL QUERY WHERE query LIKE 'SELECT URL, uniq(SearchPhrase) AS u FROM test.hits GROUP BY URL ORDER BY u %'"))
|
||||
call_with_retry(
|
||||
make_query_command(
|
||||
"KILL QUERY WHERE query LIKE 'SELECT URL, uniq(SearchPhrase) AS u FROM test.hits GROUP BY URL ORDER BY u %'"
|
||||
)
|
||||
)
|
||||
|
||||
if drop_databases:
|
||||
for i in range(5):
|
||||
@ -123,23 +152,35 @@ def prepare_for_hung_check(drop_databases):
|
||||
# Otherwise we will get rid of queries which wait for background pool. It can take a long time on slow builds (more than 900 seconds).
|
||||
#
|
||||
# Also specify max_untracked_memory to allow 1GiB of memory to overcommit.
|
||||
databases = check_output(make_query_command('SHOW DATABASES'), shell=True, timeout=30).decode('utf-8').strip().split()
|
||||
databases = (
|
||||
check_output(
|
||||
make_query_command("SHOW DATABASES"), shell=True, timeout=30
|
||||
)
|
||||
.decode("utf-8")
|
||||
.strip()
|
||||
.split()
|
||||
)
|
||||
for db in databases:
|
||||
if db == "system":
|
||||
continue
|
||||
command = make_query_command(f'DROP DATABASE {db}')
|
||||
command = make_query_command(f"DROP DATABASE {db}")
|
||||
# we don't wait for drop
|
||||
Popen(command, shell=True)
|
||||
break
|
||||
except Exception as ex:
|
||||
logging.error("Failed to SHOW or DROP databasese, will retry %s", str(ex))
|
||||
logging.error(
|
||||
"Failed to SHOW or DROP databasese, will retry %s", str(ex)
|
||||
)
|
||||
time.sleep(i)
|
||||
else:
|
||||
raise Exception("Cannot drop databases after stress tests. Probably server consumed too much memory and cannot execute simple queries")
|
||||
|
||||
raise Exception(
|
||||
"Cannot drop databases after stress tests. Probably server consumed too much memory and cannot execute simple queries"
|
||||
)
|
||||
|
||||
# Wait for last queries to finish if any, not longer than 300 seconds
|
||||
call(make_query_command("""
|
||||
call(
|
||||
make_query_command(
|
||||
"""
|
||||
select sleepEachRow((
|
||||
select maxOrDefault(300 - elapsed) + 1
|
||||
from system.processes
|
||||
@ -147,39 +188,58 @@ def prepare_for_hung_check(drop_databases):
|
||||
) / 300)
|
||||
from numbers(300)
|
||||
format Null
|
||||
"""), shell=True, stderr=STDOUT, timeout=330)
|
||||
"""
|
||||
),
|
||||
shell=True,
|
||||
stderr=STDOUT,
|
||||
timeout=330,
|
||||
)
|
||||
|
||||
# Even if all clickhouse-test processes are finished, there are probably some sh scripts,
|
||||
# which still run some new queries. Let's ignore them.
|
||||
try:
|
||||
query = """clickhouse client -q "SELECT count() FROM system.processes where where elapsed > 300" """
|
||||
output = check_output(query, shell=True, stderr=STDOUT, timeout=30).decode('utf-8').strip()
|
||||
output = (
|
||||
check_output(query, shell=True, stderr=STDOUT, timeout=30)
|
||||
.decode("utf-8")
|
||||
.strip()
|
||||
)
|
||||
if int(output) == 0:
|
||||
return False
|
||||
except:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
||||
parser = argparse.ArgumentParser(description="ClickHouse script for running stresstest")
|
||||
parser.add_argument("--test-cmd", default='/usr/bin/clickhouse-test')
|
||||
parser.add_argument("--skip-func-tests", default='')
|
||||
parser.add_argument("--client-cmd", default='clickhouse-client')
|
||||
parser.add_argument("--server-log-folder", default='/var/log/clickhouse-server')
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="ClickHouse script for running stresstest"
|
||||
)
|
||||
parser.add_argument("--test-cmd", default="/usr/bin/clickhouse-test")
|
||||
parser.add_argument("--skip-func-tests", default="")
|
||||
parser.add_argument("--client-cmd", default="clickhouse-client")
|
||||
parser.add_argument("--server-log-folder", default="/var/log/clickhouse-server")
|
||||
parser.add_argument("--output-folder")
|
||||
parser.add_argument("--global-time-limit", type=int, default=1800)
|
||||
parser.add_argument("--num-parallel", type=int, default=cpu_count())
|
||||
parser.add_argument('--backward-compatibility-check', action='store_true')
|
||||
parser.add_argument('--hung-check', action='store_true', default=False)
|
||||
parser.add_argument("--backward-compatibility-check", action="store_true")
|
||||
parser.add_argument("--hung-check", action="store_true", default=False)
|
||||
# make sense only for hung check
|
||||
parser.add_argument('--drop-databases', action='store_true', default=False)
|
||||
parser.add_argument("--drop-databases", action="store_true", default=False)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.drop_databases and not args.hung_check:
|
||||
raise Exception("--drop-databases only used in hung check (--hung-check)")
|
||||
func_pipes = []
|
||||
func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests, args.global_time_limit, args.backward_compatibility_check)
|
||||
func_pipes = run_func_test(
|
||||
args.test_cmd,
|
||||
args.output_folder,
|
||||
args.num_parallel,
|
||||
args.skip_func_tests,
|
||||
args.global_time_limit,
|
||||
args.backward_compatibility_check,
|
||||
)
|
||||
|
||||
logging.info("Will wait functests to finish")
|
||||
while True:
|
||||
@ -205,32 +265,41 @@ if __name__ == "__main__":
|
||||
have_long_running_queries = True
|
||||
logging.error("Failed to prepare for hung check %s", str(ex))
|
||||
logging.info("Checking if some queries hung")
|
||||
cmd = ' '.join([args.test_cmd,
|
||||
# Do not track memory allocations up to 1Gi,
|
||||
# this will allow to ignore server memory limit (max_server_memory_usage) for this query.
|
||||
#
|
||||
# NOTE: memory_profiler_step should be also adjusted, because:
|
||||
#
|
||||
# untracked_memory_limit = min(settings.max_untracked_memory, settings.memory_profiler_step)
|
||||
#
|
||||
# NOTE: that if there will be queries with GROUP BY, this trick
|
||||
# will not work due to CurrentMemoryTracker::check() from
|
||||
# Aggregator code.
|
||||
# But right now it should work, since neither hung check, nor 00001_select_1 has GROUP BY.
|
||||
"--client-option", "max_untracked_memory=1Gi",
|
||||
"--client-option", "max_memory_usage_for_user=0",
|
||||
"--client-option", "memory_profiler_step=1Gi",
|
||||
# Use system database to avoid CREATE/DROP DATABASE queries
|
||||
"--database=system",
|
||||
"--hung-check",
|
||||
"00001_select_1"
|
||||
])
|
||||
cmd = " ".join(
|
||||
[
|
||||
args.test_cmd,
|
||||
# Do not track memory allocations up to 1Gi,
|
||||
# this will allow to ignore server memory limit (max_server_memory_usage) for this query.
|
||||
#
|
||||
# NOTE: memory_profiler_step should be also adjusted, because:
|
||||
#
|
||||
# untracked_memory_limit = min(settings.max_untracked_memory, settings.memory_profiler_step)
|
||||
#
|
||||
# NOTE: that if there will be queries with GROUP BY, this trick
|
||||
# will not work due to CurrentMemoryTracker::check() from
|
||||
# Aggregator code.
|
||||
# But right now it should work, since neither hung check, nor 00001_select_1 has GROUP BY.
|
||||
"--client-option",
|
||||
"max_untracked_memory=1Gi",
|
||||
"--client-option",
|
||||
"max_memory_usage_for_user=0",
|
||||
"--client-option",
|
||||
"memory_profiler_step=1Gi",
|
||||
# Use system database to avoid CREATE/DROP DATABASE queries
|
||||
"--database=system",
|
||||
"--hung-check",
|
||||
"--stress",
|
||||
"00001_select_1",
|
||||
]
|
||||
)
|
||||
res = call(cmd, shell=True, stderr=STDOUT)
|
||||
hung_check_status = "No queries hung\tOK\n"
|
||||
if res != 0 and have_long_running_queries:
|
||||
logging.info("Hung check failed with exit code {}".format(res))
|
||||
hung_check_status = "Hung check failed\tFAIL\n"
|
||||
with open(os.path.join(args.output_folder, "test_results.tsv"), 'w+') as results:
|
||||
with open(
|
||||
os.path.join(args.output_folder, "test_results.tsv"), "w+"
|
||||
) as results:
|
||||
results.write(hung_check_status)
|
||||
|
||||
logging.info("Stress test finished")
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
* Selects with final are executed in parallel. Added setting `max_final_threads` to limit the number of threads used. [#10463](https://github.com/ClickHouse/ClickHouse/pull/10463) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Function that extracts from haystack all matching non-overlapping groups with regular expressions, and put those into `Array(Array(String))` column. [#10534](https://github.com/ClickHouse/ClickHouse/pull/10534) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Added ability to delete a subset of expired rows, which satisfies the condition in WHERE clause. Added ability to replace expired rows with aggregates of them specified in GROUP BY clause. [#10537](https://github.com/ClickHouse/ClickHouse/pull/10537) ([expl0si0nn](https://github.com/expl0si0nn)).
|
||||
* (Only Linux) Clickhouse server now tries to fallback to ProcfsMetricsProvider when clickhouse binary is not attributed with CAP_NET_ADMIN capability to collect per-query system metrics (for CPU and I/O). [#10544](https://github.com/ClickHouse/ClickHouse/pull/10544) ([Alexander Kazakov](https://github.com/Akazz)).
|
||||
* (Only Linux) ClickHouse server now tries to fallback to ProcfsMetricsProvider when clickhouse binary is not attributed with CAP_NET_ADMIN capability to collect per-query system metrics (for CPU and I/O). [#10544](https://github.com/ClickHouse/ClickHouse/pull/10544) ([Alexander Kazakov](https://github.com/Akazz)).
|
||||
* - Add Arrow IPC File format (Input and Output) - Fix incorrect work of resetParser() for Parquet Input Format - Add zero-copy optimization for ORC for RandomAccessFiles - Add missing halffloat type for input parquet and ORC formats ... [#10580](https://github.com/ClickHouse/ClickHouse/pull/10580) ([Zhanna](https://github.com/FawnD2)).
|
||||
* Allowed to profile memory with finer granularity steps than 4 MiB. Added sampling memory profiler to capture random allocations/deallocations. [#10598](https://github.com/ClickHouse/ClickHouse/pull/10598) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Add new input format `JSONAsString` that accepts a sequence of JSON objects separated by newlines, spaces and/or commas. [#10607](https://github.com/ClickHouse/ClickHouse/pull/10607) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
* Users now can set comments to database in `CREATE DATABASE` statement ... [#29429](https://github.com/ClickHouse/ClickHouse/pull/29429) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* New function` mapContainsKeyLike` to get the map that key matches a simple regular expression. [#29471](https://github.com/ClickHouse/ClickHouse/pull/29471) ([凌涛](https://github.com/lingtaolf)).
|
||||
* Huawei OBS Storage support. Closes [#24294](https://github.com/ClickHouse/ClickHouse/issues/24294). [#29511](https://github.com/ClickHouse/ClickHouse/pull/29511) ([kevin wan](https://github.com/MaxWk)).
|
||||
* Clickhouse HTTP Server can enable HSTS by set `hsts_max_age` in config.xml with a positive number. [#29516](https://github.com/ClickHouse/ClickHouse/pull/29516) ([凌涛](https://github.com/lingtaolf)).
|
||||
* ClickHouse HTTP Server can enable HSTS by set `hsts_max_age` in config.xml with a positive number. [#29516](https://github.com/ClickHouse/ClickHouse/pull/29516) ([凌涛](https://github.com/lingtaolf)).
|
||||
* - Added MD4 and SHA384 functions. [#29602](https://github.com/ClickHouse/ClickHouse/pull/29602) ([Nikita Tikhomirov](https://github.com/NSTikhomirov)).
|
||||
* Support EXISTS(subquery). Closes [#6852](https://github.com/ClickHouse/ClickHouse/issues/6852). [#29731](https://github.com/ClickHouse/ClickHouse/pull/29731) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Added function `ngram`. Closes [#29699](https://github.com/ClickHouse/ClickHouse/issues/29699). [#29738](https://github.com/ClickHouse/ClickHouse/pull/29738) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
|
||||
@ -54,7 +54,7 @@
|
||||
* Add settings `merge_tree_min_rows_for_concurrent_read_for_remote_filesystem` and `merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem`. [#30970](https://github.com/ClickHouse/ClickHouse/pull/30970) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Do not allow to drop a table or dictionary if some tables or dictionaries depend on it. [#30977](https://github.com/ClickHouse/ClickHouse/pull/30977) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Only grab AlterLock when we do alter command. Let's see if the assumption is correct. [#31010](https://github.com/ClickHouse/ClickHouse/pull/31010) ([Amos Bird](https://github.com/amosbird)).
|
||||
* The local session inside a Clickhouse dictionary source won't send its events to the session log anymore. This fixes a possible deadlock (tsan alert) on shutdown. Also this PR fixes flaky `test_dictionaries_dependency_xml/`. [#31013](https://github.com/ClickHouse/ClickHouse/pull/31013) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* The local session inside a ClickHouse dictionary source won't send its events to the session log anymore. This fixes a possible deadlock (tsan alert) on shutdown. Also this PR fixes flaky `test_dictionaries_dependency_xml/`. [#31013](https://github.com/ClickHouse/ClickHouse/pull/31013) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Cancel vertical merges when partition is dropped. This is a follow-up of https://github.com/ClickHouse/ClickHouse/pull/25684 and https://github.com/ClickHouse/ClickHouse/pull/30996. [#31057](https://github.com/ClickHouse/ClickHouse/pull/31057) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Support `IF EXISTS` modifier for `RENAME DATABASE`/`TABLE`/`DICTIONARY` query, If this directive is used, one will not get an error if the DATABASE/TABLE/DICTIONARY to be renamed doesn't exist. [#31081](https://github.com/ClickHouse/ClickHouse/pull/31081) ([victorgao](https://github.com/kafka1991)).
|
||||
* Function name normalization for ALTER queries. This helps avoid metadata mismatch between creating table with indices/projections and adding indices/projections via alter commands. This is a follow-up PR of https://github.com/ClickHouse/ClickHouse/pull/20174. Mark as improvements as there are no bug reports and the senario is somehow rare. [#31095](https://github.com/ClickHouse/ClickHouse/pull/31095) ([Amos Bird](https://github.com/amosbird)).
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
### ClickHouse release v21.12.3.32-stable FIXME as compared to v21.12.2.17-stable
|
||||
|
||||
#### Bug Fix
|
||||
* Backported in [#33018](https://github.com/ClickHouse/ClickHouse/issues/33018): - Clickhouse Keeper handler should remove operation when response sent. [#32988](https://github.com/ClickHouse/ClickHouse/pull/32988) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Backported in [#33018](https://github.com/ClickHouse/ClickHouse/issues/33018): - ClickHouse Keeper handler should remove operation when response sent. [#32988](https://github.com/ClickHouse/ClickHouse/pull/32988) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
|
||||
|
||||
@ -68,7 +68,7 @@
|
||||
* Add separate pool for message brokers (RabbitMQ and Kafka). [#19722](https://github.com/ClickHouse/ClickHouse/pull/19722) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* In distributed queries if the setting `async_socket_for_remote` is enabled, it was possible to get stack overflow at least in debug build configuration if very deeply nested data type is used in table (e.g. `Array(Array(Array(...more...)))`). This fixes [#19108](https://github.com/ClickHouse/ClickHouse/issues/19108). This change introduces minor backward incompatibility: excessive parenthesis in type definitions no longer supported, example: `Array((UInt8))`. [#19736](https://github.com/ClickHouse/ClickHouse/pull/19736) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Table function `S3` will use global region if the region can't be determined exactly. This closes [#10998](https://github.com/ClickHouse/ClickHouse/issues/10998). [#19750](https://github.com/ClickHouse/ClickHouse/pull/19750) ([Vladimir Chebotarev](https://github.com/excitoon)).
|
||||
* Clickhouse client query param CTE added test. [#19762](https://github.com/ClickHouse/ClickHouse/pull/19762) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* ClickHouse client query param CTE added test. [#19762](https://github.com/ClickHouse/ClickHouse/pull/19762) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Correctly output infinite arguments for `formatReadableTimeDelta` function. In previous versions, there was implicit conversion to implementation specific integer value. [#19791](https://github.com/ClickHouse/ClickHouse/pull/19791) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* `S3` table function now supports `auto` compression mode (autodetect). This closes [#18754](https://github.com/ClickHouse/ClickHouse/issues/18754). [#19793](https://github.com/ClickHouse/ClickHouse/pull/19793) ([Vladimir Chebotarev](https://github.com/excitoon)).
|
||||
* Set charset to utf8mb4 when interacting with remote MySQL servers. Fixes [#19795](https://github.com/ClickHouse/ClickHouse/issues/19795). [#19800](https://github.com/ClickHouse/ClickHouse/pull/19800) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
@ -34,7 +34,7 @@
|
||||
* Allow to use CTE in VIEW definition. This closes [#22491](https://github.com/ClickHouse/ClickHouse/issues/22491). [#22657](https://github.com/ClickHouse/ClickHouse/pull/22657) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Add metric to track how much time is spend during waiting for Buffer layer lock. [#22725](https://github.com/ClickHouse/ClickHouse/pull/22725) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Allow RBAC row policy via postgresql protocol. Closes [#22658](https://github.com/ClickHouse/ClickHouse/issues/22658). PostgreSQL protocol is enabled in configuration by default. [#22755](https://github.com/ClickHouse/ClickHouse/pull/22755) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* MaterializeMySQL (experimental feature). Make Clickhouse to be able to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. ... [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian Frøystad](https://github.com/cfroystad)).
|
||||
* MaterializeMySQL (experimental feature). Make ClickHouse to be able to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. ... [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian Frøystad](https://github.com/cfroystad)).
|
||||
* `dateDiff` now works with `DateTime64` arguments (even for values outside of `DateTime` range) ... [#22931](https://github.com/ClickHouse/ClickHouse/pull/22931) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix inactive_parts_to_throw_insert=0 with inactive_parts_to_delay_insert>0. [#22947](https://github.com/ClickHouse/ClickHouse/pull/22947) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
@ -84,7 +84,7 @@
|
||||
|
||||
#### Bug Fix
|
||||
* Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)).
|
||||
* - Clickhouse Keeper handler should remove operation when response sent. [#32988](https://github.com/ClickHouse/ClickHouse/pull/32988) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* - ClickHouse Keeper handler should remove operation when response sent. [#32988](https://github.com/ClickHouse/ClickHouse/pull/32988) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Fix null pointer dereference in low cardinality data when deserializing LowCardinality data in the Native format. [#33021](https://github.com/ClickHouse/ClickHouse/pull/33021) ([Harry Lee](https://github.com/HarryLeeIBM)).
|
||||
* Specifically crafted input data for `Native` format may lead to reading uninitialized memory or crash. This is relevant if `clickhouse-server` is open for write access to adversary. [#33050](https://github.com/ClickHouse/ClickHouse/pull/33050) ([Heena Bansal](https://github.com/HeenaBansal2009)).
|
||||
|
||||
@ -196,7 +196,7 @@
|
||||
* NO CL ENTRY: 'Update CHANGELOG.md'. [#32472](https://github.com/ClickHouse/ClickHouse/pull/32472) ([Rich Raposa](https://github.com/rfraposa)).
|
||||
* NO CL ENTRY: 'Revert "Split long tests into multiple checks"'. [#32514](https://github.com/ClickHouse/ClickHouse/pull/32514) ([alesapin](https://github.com/alesapin)).
|
||||
* NO CL ENTRY: 'Revert "Revert "Split long tests into multiple checks""'. [#32515](https://github.com/ClickHouse/ClickHouse/pull/32515) ([alesapin](https://github.com/alesapin)).
|
||||
* NO CL ENTRY: 'blog post how to enable predictive capabilities in Clickhouse'. [#32768](https://github.com/ClickHouse/ClickHouse/pull/32768) ([Tom Risse](https://github.com/flickerbox-tom)).
|
||||
* NO CL ENTRY: 'blog post how to enable predictive capabilities in ClickHouse'. [#32768](https://github.com/ClickHouse/ClickHouse/pull/32768) ([Tom Risse](https://github.com/flickerbox-tom)).
|
||||
* NO CL ENTRY: 'Revert "Fix build issue related to azure blob storage"'. [#32845](https://github.com/ClickHouse/ClickHouse/pull/32845) ([alesapin](https://github.com/alesapin)).
|
||||
* NO CL ENTRY: 'Revert "Dictionaries added Date32 type support"'. [#33053](https://github.com/ClickHouse/ClickHouse/pull/33053) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* NO CL ENTRY: 'Updated Lawrence Berkeley National Lab stats'. [#33066](https://github.com/ClickHouse/ClickHouse/pull/33066) ([Michael Smitasin](https://github.com/michaelsmitasin)).
|
||||
|
||||
@ -65,7 +65,7 @@
|
||||
* Add setting to lower column case when reading parquet/ORC file. [#35145](https://github.com/ClickHouse/ClickHouse/pull/35145) ([shuchaome](https://github.com/shuchaome)).
|
||||
* Do not retry non-rertiable errors. Closes [#35161](https://github.com/ClickHouse/ClickHouse/issues/35161). [#35172](https://github.com/ClickHouse/ClickHouse/pull/35172) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Added disk_name to system.part_log. [#35178](https://github.com/ClickHouse/ClickHouse/pull/35178) ([Artyom Yurkov](https://github.com/Varinara)).
|
||||
* Currently,Clickhouse validates hosts defined under <remote_url_allow_hosts> for URL and Remote Table functions. This PR extends the RemoteHostFilter to Mysql and PostgreSQL table functions. [#35191](https://github.com/ClickHouse/ClickHouse/pull/35191) ([Heena Bansal](https://github.com/HeenaBansal2009)).
|
||||
* Currently,ClickHouse validates hosts defined under <remote_url_allow_hosts> for URL and Remote Table functions. This PR extends the RemoteHostFilter to Mysql and PostgreSQL table functions. [#35191](https://github.com/ClickHouse/ClickHouse/pull/35191) ([Heena Bansal](https://github.com/HeenaBansal2009)).
|
||||
* Sometimes it is not enough for us to distinguish queries hierachy only by is_initial_query in system.query_log and system.processes. So distributed_depth is needed. [#35207](https://github.com/ClickHouse/ClickHouse/pull/35207) ([李扬](https://github.com/taiyang-li)).
|
||||
* Support test mode for clickhouse-local. [#35264](https://github.com/ClickHouse/ClickHouse/pull/35264) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Return const for function getMacro if not in distributed query. Close [#34727](https://github.com/ClickHouse/ClickHouse/issues/34727). [#35289](https://github.com/ClickHouse/ClickHouse/pull/35289) ([李扬](https://github.com/taiyang-li)).
|
||||
|
||||
@ -97,7 +97,7 @@ SELECT library_name, license_type, license_path FROM system.licenses ORDER BY li
|
||||
## Adding new third-party libraries and maintaining patches in third-party libraries {#adding-third-party-libraries}
|
||||
|
||||
1. Each third-party library must reside in a dedicated directory under the `contrib/` directory of the ClickHouse repository. Avoid dumps/copies of external code, instead use Git submodule feature to pull third-party code from an external upstream repository.
|
||||
2. Submodules are listed in `.gitmodule`. If the external library can be used as-is, you may reference the upstream repository directly. Otherwise, i.e. the external library requires patching/customization, create a fork of the official repository in the [Clickhouse organization in GitHub](https://github.com/ClickHouse).
|
||||
2. Submodules are listed in `.gitmodule`. If the external library can be used as-is, you may reference the upstream repository directly. Otherwise, i.e. the external library requires patching/customization, create a fork of the official repository in the [ClickHouse organization in GitHub](https://github.com/ClickHouse).
|
||||
3. In the latter case, create a branch with `clickhouse/` prefix from the branch you want to integrate, e.g. `clickhouse/master` (for `master`) or `clickhouse/release/vX.Y.Z` (for a `release/vX.Y.Z` tag). The purpose of this branch is to isolate customization of the library from upstream work. For example, pulls from the upstream repository into the fork will leave all `clickhouse/` branches unaffected. Submodules in `contrib/` must only track `clickhouse/` branches of forked third-party repositories.
|
||||
4. To patch a fork of a third-party library, create a dedicated branch with `clickhouse/` prefix in the fork, e.g. `clickhouse/fix-some-desaster`. Finally, merge the patch branch into the custom tracking branch (e.g. `clickhouse/master` or `clickhouse/release/vX.Y.Z`) using a PR.
|
||||
5. Always create patches of third-party libraries with the official repository in mind. Once a PR of a patch branch to the `clickhouse/` branch in the fork repository is done and the submodule version in ClickHouse official repository is bumped, consider opening another PR from the patch branch to the upstream library repository. This ensures, that 1) the contribution has more than a single use case and importance, 2) others will also benefit from it, 3) the change will not remain a maintenance burden solely on ClickHouse developers.
|
||||
|
||||
@ -66,7 +66,7 @@ For a description of parameters, see the [CREATE query description](../../../sql
|
||||
|
||||
A tuple of column names or arbitrary expressions. Example: `ORDER BY (CounterID, EventDate)`.
|
||||
|
||||
ClickHouse uses the sorting key as a primary key if the primary key is not defined obviously by the `PRIMARY KEY` clause.
|
||||
ClickHouse uses the sorting key as a primary key if the primary key is not defined explicitly by the `PRIMARY KEY` clause.
|
||||
|
||||
Use the `ORDER BY tuple()` syntax, if you do not need sorting. See [Selecting the Primary Key](#selecting-the-primary-key).
|
||||
|
||||
|
||||
@ -27,7 +27,7 @@ Compressed data for `INSERT` and `ALTER` queries is replicated (for more informa
|
||||
- The `DROP TABLE` query deletes the replica located on the server where the query is run.
|
||||
- The `RENAME` query renames the table on one of the replicas. In other words, replicated tables can have different names on different replicas.
|
||||
|
||||
ClickHouse uses [Apache ZooKeeper](https://zookeeper.apache.org) for storing replicas meta information. Use ZooKeeper version 3.4.5 or newer.
|
||||
ClickHouse uses [ClickHouse Keeper](../../../guides/sre/keeper/clickhouse-keeper.md) for storing replicas meta information. It is possible to use ZooKeeper version 3.4.5 or newer, but ClickHouse Keeper is recommended.
|
||||
|
||||
To use replication, set parameters in the [zookeeper](../../../operations/server-configuration-parameters/settings.md#server-settings_zookeeper) server configuration section.
|
||||
|
||||
@ -35,7 +35,7 @@ To use replication, set parameters in the [zookeeper](../../../operations/server
|
||||
Don’t neglect the security setting. ClickHouse supports the `digest` [ACL scheme](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) of the ZooKeeper security subsystem.
|
||||
:::
|
||||
|
||||
Example of setting the addresses of the ZooKeeper cluster:
|
||||
Example of setting the addresses of the ClickHouse Keeper cluster:
|
||||
|
||||
``` xml
|
||||
<zookeeper>
|
||||
@ -54,8 +54,8 @@ Example of setting the addresses of the ZooKeeper cluster:
|
||||
</zookeeper>
|
||||
```
|
||||
|
||||
ClickHouse also supports to store replicas meta information in the auxiliary ZooKeeper cluster by providing ZooKeeper cluster name and path as engine arguments.
|
||||
In other word, it supports to store the metadata of differnt tables in different ZooKeeper clusters.
|
||||
ClickHouse also supports storing replicas meta information in an auxiliary ZooKeeper cluster. Do this by providing the ZooKeeper cluster name and path as engine arguments.
|
||||
In other words, it supports storing the metadata of different tables in different ZooKeeper clusters.
|
||||
|
||||
Example of setting the addresses of the auxiliary ZooKeeper cluster:
|
||||
|
||||
@ -122,8 +122,8 @@ The `Replicated` prefix is added to the table engine name. For example:`Replicat
|
||||
|
||||
**Replicated\*MergeTree parameters**
|
||||
|
||||
- `zoo_path` — The path to the table in ZooKeeper.
|
||||
- `replica_name` — The replica name in ZooKeeper.
|
||||
- `zoo_path` — The path to the table in ClickHouse Keeper.
|
||||
- `replica_name` — The replica name in ClickHouse Keeper.
|
||||
- `other_parameters` — Parameters of an engine which is used for creating the replicated version, for example, version in `ReplacingMergeTree`.
|
||||
|
||||
Example:
|
||||
@ -168,18 +168,18 @@ Example:
|
||||
</macros>
|
||||
```
|
||||
|
||||
The path to the table in ZooKeeper should be unique for each replicated table. Tables on different shards should have different paths.
|
||||
The path to the table in ClickHouse Keeper should be unique for each replicated table. Tables on different shards should have different paths.
|
||||
In this case, the path consists of the following parts:
|
||||
|
||||
`/clickhouse/tables/` is the common prefix. We recommend using exactly this one.
|
||||
|
||||
`{layer}-{shard}` is the shard identifier. In this example it consists of two parts, since the example cluster uses bi-level sharding. For most tasks, you can leave just the {shard} substitution, which will be expanded to the shard identifier.
|
||||
|
||||
`table_name` is the name of the node for the table in ZooKeeper. It is a good idea to make it the same as the table name. It is defined explicitly, because in contrast to the table name, it does not change after a RENAME query.
|
||||
`table_name` is the name of the node for the table in ClickHouse Keeper. It is a good idea to make it the same as the table name. It is defined explicitly, because in contrast to the table name, it does not change after a RENAME query.
|
||||
*HINT*: you could add a database name in front of `table_name` as well. E.g. `db_name.table_name`
|
||||
|
||||
The two built-in substitutions `{database}` and `{table}` can be used, they expand into the table name and the database name respectively (unless these macros are defined in the `macros` section). So the zookeeper path can be specified as `'/clickhouse/tables/{layer}-{shard}/{database}/{table}'`.
|
||||
Be careful with table renames when using these built-in substitutions. The path in Zookeeper cannot be changed, and when the table is renamed, the macros will expand into a different path, the table will refer to a path that does not exist in Zookeeper, and will go into read-only mode.
|
||||
Be careful with table renames when using these built-in substitutions. The path in ClickHouse Keeper cannot be changed, and when the table is renamed, the macros will expand into a different path, the table will refer to a path that does not exist in ClickHouse Keeper, and will go into read-only mode.
|
||||
|
||||
The replica name identifies different replicas of the same table. You can use the server name for this, as in the example. The name only needs to be unique within each shard.
|
||||
|
||||
@ -220,21 +220,21 @@ To delete a replica, run `DROP TABLE`. However, only one replica is deleted –
|
||||
|
||||
## Recovery After Failures {#recovery-after-failures}
|
||||
|
||||
If ZooKeeper is unavailable when a server starts, replicated tables switch to read-only mode. The system periodically attempts to connect to ZooKeeper.
|
||||
If ClickHouse Keeper is unavailable when a server starts, replicated tables switch to read-only mode. The system periodically attempts to connect to ClickHouse Keeper.
|
||||
|
||||
If ZooKeeper is unavailable during an `INSERT`, or an error occurs when interacting with ZooKeeper, an exception is thrown.
|
||||
If ClickHouse Keeper is unavailable during an `INSERT`, or an error occurs when interacting with ClickHouse Keeper, an exception is thrown.
|
||||
|
||||
After connecting to ZooKeeper, the system checks whether the set of data in the local file system matches the expected set of data (ZooKeeper stores this information). If there are minor inconsistencies, the system resolves them by syncing data with the replicas.
|
||||
After connecting to ClickHouse Keeper, the system checks whether the set of data in the local file system matches the expected set of data (ClickHouse Keeper stores this information). If there are minor inconsistencies, the system resolves them by syncing data with the replicas.
|
||||
|
||||
If the system detects broken data parts (with the wrong size of files) or unrecognized parts (parts written to the file system but not recorded in ZooKeeper), it moves them to the `detached` subdirectory (they are not deleted). Any missing parts are copied from the replicas.
|
||||
If the system detects broken data parts (with the wrong size of files) or unrecognized parts (parts written to the file system but not recorded in ClickHouse Keeper), it moves them to the `detached` subdirectory (they are not deleted). Any missing parts are copied from the replicas.
|
||||
|
||||
Note that ClickHouse does not perform any destructive actions such as automatically deleting a large amount of data.
|
||||
|
||||
When the server starts (or establishes a new session with ZooKeeper), it only checks the quantity and sizes of all files. If the file sizes match but bytes have been changed somewhere in the middle, this is not detected immediately, but only when attempting to read the data for a `SELECT` query. The query throws an exception about a non-matching checksum or size of a compressed block. In this case, data parts are added to the verification queue and copied from the replicas if necessary.
|
||||
When the server starts (or establishes a new session with ClickHouse Keeper), it only checks the quantity and sizes of all files. If the file sizes match but bytes have been changed somewhere in the middle, this is not detected immediately, but only when attempting to read the data for a `SELECT` query. The query throws an exception about a non-matching checksum or size of a compressed block. In this case, data parts are added to the verification queue and copied from the replicas if necessary.
|
||||
|
||||
If the local set of data differs too much from the expected one, a safety mechanism is triggered. The server enters this in the log and refuses to launch. The reason for this is that this case may indicate a configuration error, such as if a replica on a shard was accidentally configured like a replica on a different shard. However, the thresholds for this mechanism are set fairly low, and this situation might occur during normal failure recovery. In this case, data is restored semi-automatically - by “pushing a button”.
|
||||
|
||||
To start recovery, create the node `/path_to_table/replica_name/flags/force_restore_data` in ZooKeeper with any content, or run the command to restore all replicated tables:
|
||||
To start recovery, create the node `/path_to_table/replica_name/flags/force_restore_data` in ClickHouse Keeper with any content, or run the command to restore all replicated tables:
|
||||
|
||||
``` bash
|
||||
sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data
|
||||
@ -249,11 +249,11 @@ If all data and metadata disappeared from one of the servers, follow these steps
|
||||
1. Install ClickHouse on the server. Define substitutions correctly in the config file that contains the shard identifier and replicas, if you use them.
|
||||
2. If you had unreplicated tables that must be manually duplicated on the servers, copy their data from a replica (in the directory `/var/lib/clickhouse/data/db_name/table_name/`).
|
||||
3. Copy table definitions located in `/var/lib/clickhouse/metadata/` from a replica. If a shard or replica identifier is defined explicitly in the table definitions, correct it so that it corresponds to this replica. (Alternatively, start the server and make all the `ATTACH TABLE` queries that should have been in the .sql files in `/var/lib/clickhouse/metadata/`.)
|
||||
4. To start recovery, create the ZooKeeper node `/path_to_table/replica_name/flags/force_restore_data` with any content, or run the command to restore all replicated tables: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data`
|
||||
4. To start recovery, create the ClickHouse Keeper node `/path_to_table/replica_name/flags/force_restore_data` with any content, or run the command to restore all replicated tables: `sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data`
|
||||
|
||||
Then start the server (restart, if it is already running). Data will be downloaded from replicas.
|
||||
|
||||
An alternative recovery option is to delete information about the lost replica from ZooKeeper (`/path_to_table/replica_name`), then create the replica again as described in “[Creating replicated tables](#creating-replicated-tables)”.
|
||||
An alternative recovery option is to delete information about the lost replica from ClickHouse Keeper (`/path_to_table/replica_name`), then create the replica again as described in “[Creating replicated tables](#creating-replicated-tables)”.
|
||||
|
||||
There is no restriction on network bandwidth during recovery. Keep this in mind if you are restoring many replicas at once.
|
||||
|
||||
@ -276,13 +276,13 @@ Create a MergeTree table with a different name. Move all the data from the direc
|
||||
If you want to get rid of a `ReplicatedMergeTree` table without launching the server:
|
||||
|
||||
- Delete the corresponding `.sql` file in the metadata directory (`/var/lib/clickhouse/metadata/`).
|
||||
- Delete the corresponding path in ZooKeeper (`/path_to_table/replica_name`).
|
||||
- Delete the corresponding path in ClickHouse Keeper (`/path_to_table/replica_name`).
|
||||
|
||||
After this, you can launch the server, create a `MergeTree` table, move the data to its directory, and then restart the server.
|
||||
|
||||
## Recovery When Metadata in the Zookeeper Cluster Is Lost or Damaged {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged}
|
||||
## Recovery When Metadata in the ClickHouse Keeper Cluster Is Lost or Damaged {#recovery-when-metadata-in-the-zookeeper-cluster-is-lost-or-damaged}
|
||||
|
||||
If the data in ZooKeeper was lost or damaged, you can save data by moving it to an unreplicated table as described above.
|
||||
If the data in ClickHouse Keeper was lost or damaged, you can save data by moving it to an unreplicated table as described above.
|
||||
|
||||
**See Also**
|
||||
|
||||
|
||||
@ -5,6 +5,8 @@ sidebar_label: Command-Line Client
|
||||
|
||||
# Command-line Client
|
||||
|
||||
## clickhouse-client
|
||||
|
||||
ClickHouse provides a native command-line client: `clickhouse-client`. The client supports command-line options and configuration files. For more information, see [Configuring](#interfaces_cli_configuration).
|
||||
|
||||
[Install](../getting-started/install.md) it from the `clickhouse-client` package and run it with the command `clickhouse-client`.
|
||||
@ -115,7 +117,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
|
||||
- `--user, -u` – The username. Default value: default.
|
||||
- `--password` – The password. Default value: empty string.
|
||||
- `--query, -q` – The query to process when using non-interactive mode. You must specify either `query` or `queries-file` option.
|
||||
- `--queries-file, -qf` – file path with queries to execute. You must specify either `query` or `queries-file` option.
|
||||
- `--queries-file` – file path with queries to execute. You must specify either `query` or `queries-file` option.
|
||||
- `--database, -d` – Select the current default database. Default value: the current database from the server settings (‘default’ by default).
|
||||
- `--multiline, -m` – If specified, allow multiline queries (do not send the query on Enter).
|
||||
- `--multiquery, -n` – If specified, allow processing multiple queries separated by semicolons.
|
||||
@ -183,4 +185,3 @@ If the configuration above is applied, the ID of a query is shown in the followi
|
||||
``` text
|
||||
speedscope:http://speedscope-host/#profileURL=qp%3Fid%3Dc8ecc783-e753-4b38-97f1-42cddfb98b7d
|
||||
```
|
||||
|
||||
|
||||
@ -7,4 +7,6 @@ sidebar_label: C++ Client Library
|
||||
|
||||
See README at [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp) repository.
|
||||
|
||||
[Original article](https://clickhouse.com/docs/en/interfaces/cpp/) <!--hide-->
|
||||
# userver Asynchronous Framework
|
||||
|
||||
[userver (beta)](https://github.com/userver-framework/userver) has builtin support for ClickHouse.
|
||||
|
||||
@ -28,6 +28,9 @@ ClickHouse, Inc. does **not** maintain the tools and libraries listed below and
|
||||
- [Kafka](https://kafka.apache.org)
|
||||
- [clickhouse_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/ClickHouse/clickhouse-go/))
|
||||
- [stream-loader-clickhouse](https://github.com/adform/stream-loader)
|
||||
- Batch processing
|
||||
- [Spark](https://spark.apache.org)
|
||||
- [spark-clickhouse-connector](https://github.com/housepower/spark-clickhouse-connector)
|
||||
- Stream processing
|
||||
- [Flink](https://flink.apache.org)
|
||||
- [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink)
|
||||
|
||||
@ -325,14 +325,14 @@ clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 --
|
||||
|
||||
## Recovering after losing quorum
|
||||
|
||||
Because Clickhouse Keeper uses Raft it can tolerate certain amount of node crashes depending on the cluster size. \
|
||||
Because ClickHouse Keeper uses Raft it can tolerate certain amount of node crashes depending on the cluster size. \
|
||||
E.g. for a 3-node cluster, it will continue working correctly if only 1 node crashes.
|
||||
|
||||
Cluster configuration can be dynamically configured but there are some limitations. Reconfiguration relies on Raft also
|
||||
so to add/remove a node from the cluster you need to have a quorum. If you lose too many nodes in your cluster at the same time without any chance
|
||||
of starting them again, Raft will stop working and not allow you to reconfigure your cluster using the conventional way.
|
||||
|
||||
Nevertheless, Clickhouse Keeper has a recovery mode which allows you to forcefully reconfigure your cluster with only 1 node.
|
||||
Nevertheless, ClickHouse Keeper has a recovery mode which allows you to forcefully reconfigure your cluster with only 1 node.
|
||||
This should be done only as your last resort if you cannot start your nodes again, or start a new instance on the same endpoint.
|
||||
|
||||
Important things to note before continuing:
|
||||
|
||||
@ -34,7 +34,7 @@ Example of configuration:
|
||||
<named_collections>
|
||||
<s3_mydata>
|
||||
<access_key_id>AKIAIOSFODNN7EXAMPLE</access_key_id>
|
||||
<secret_access_key> wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY</secret_access_key>
|
||||
<secret_access_key>wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY</secret_access_key>
|
||||
<format>CSV</format>
|
||||
<url>https://s3.us-east-1.amazonaws.com/yourbucket/mydata/</url>
|
||||
</s3_mydata>
|
||||
@ -227,4 +227,4 @@ SELECT dictGet('dict', 'b', 2);
|
||||
┌─dictGet('dict', 'b', 2)─┐
|
||||
│ two │
|
||||
└─────────────────────────┘
|
||||
```
|
||||
```
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# replication_queue
|
||||
|
||||
Contains information about tasks from replication queues stored in Clickhouse Keeper, or ZooKeeper, for tables in the `ReplicatedMergeTree` family.
|
||||
Contains information about tasks from replication queues stored in ClickHouse Keeper, or ZooKeeper, for tables in the `ReplicatedMergeTree` family.
|
||||
|
||||
Columns:
|
||||
|
||||
|
||||
@ -274,6 +274,6 @@ end script
|
||||
|
||||
## Antivirus software {#antivirus-software}
|
||||
|
||||
If you use antivirus software configure it to skip folders with Clickhouse datafiles (`/var/lib/clickhouse`) otherwise performance may be reduced and you may experience unexpected errors during data ingestion and background merges.
|
||||
If you use antivirus software configure it to skip folders with ClickHouse datafiles (`/var/lib/clickhouse`) otherwise performance may be reduced and you may experience unexpected errors during data ingestion and background merges.
|
||||
|
||||
[Original article](https://clickhouse.com/docs/en/operations/tips/)
|
||||
|
||||
@ -31,12 +31,12 @@ $ clickhouse-local --structure "table_structure" --input-format "format_of_incom
|
||||
Arguments:
|
||||
|
||||
- `-S`, `--structure` — table structure for input data.
|
||||
- `-if`, `--input-format` — input format, `TSV` by default.
|
||||
- `--input-format` — input format, `TSV` by default.
|
||||
- `-f`, `--file` — path to data, `stdin` by default.
|
||||
- `-q`, `--query` — queries to execute with `;` as delimeter. You must specify either `query` or `queries-file` option.
|
||||
- `-qf`, `--queries-file` - file path with queries to execute. You must specify either `query` or `queries-file` option.
|
||||
- `--queries-file` - file path with queries to execute. You must specify either `query` or `queries-file` option.
|
||||
- `-N`, `--table` — table name where to put output data, `table` by default.
|
||||
- `-of`, `--format`, `--output-format` — output format, `TSV` by default.
|
||||
- `--format`, `--output-format` — output format, `TSV` by default.
|
||||
- `-d`, `--database` — default database, `_local` by default.
|
||||
- `--stacktrace` — whether to dump debug output in case of exception.
|
||||
- `--echo` — print query before execution.
|
||||
|
||||
@ -19,11 +19,10 @@ This function encrypts data using these modes:
|
||||
|
||||
- aes-128-ecb, aes-192-ecb, aes-256-ecb
|
||||
- aes-128-cbc, aes-192-cbc, aes-256-cbc
|
||||
- aes-128-cfb1, aes-192-cfb1, aes-256-cfb1
|
||||
- aes-128-cfb8, aes-192-cfb8, aes-256-cfb8
|
||||
- aes-128-cfb128, aes-192-cfb128, aes-256-cfb128
|
||||
- aes-128-cfb128
|
||||
- aes-128-ofb, aes-192-ofb, aes-256-ofb
|
||||
- aes-128-gcm, aes-192-gcm, aes-256-gcm
|
||||
- aes-128-ctr, aes-192-ctr, aes-256-ctr
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -63,9 +62,9 @@ Insert some data (please avoid storing the keys/ivs in the database as this unde
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
INSERT INTO encryption_test VALUES('aes-256-cfb128 no IV', encrypt('aes-256-cfb128', 'Secret', '12345678910121314151617181920212')),\
|
||||
('aes-256-cfb128 no IV, different key', encrypt('aes-256-cfb128', 'Secret', 'keykeykeykeykeykeykeykeykeykeyke')),\
|
||||
('aes-256-cfb128 with IV', encrypt('aes-256-cfb128', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv')),\
|
||||
INSERT INTO encryption_test VALUES('aes-256-ofb no IV', encrypt('aes-256-ofb', 'Secret', '12345678910121314151617181920212')),\
|
||||
('aes-256-ofb no IV, different key', encrypt('aes-256-ofb', 'Secret', 'keykeykeykeykeykeykeykeykeykeyke')),\
|
||||
('aes-256-ofb with IV', encrypt('aes-256-ofb', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv')),\
|
||||
('aes-256-cbc no IV', encrypt('aes-256-cbc', 'Secret', '12345678910121314151617181920212'));
|
||||
```
|
||||
|
||||
@ -78,12 +77,12 @@ SELECT comment, hex(secret) FROM encryption_test;
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─comment─────────────────────────────┬─hex(secret)──────────────────────┐
|
||||
│ aes-256-cfb128 no IV │ B4972BDC4459 │
|
||||
│ aes-256-cfb128 no IV, different key │ 2FF57C092DC9 │
|
||||
│ aes-256-cfb128 with IV │ 5E6CB398F653 │
|
||||
│ aes-256-cbc no IV │ 1BC0629A92450D9E73A00E7D02CF4142 │
|
||||
└─────────────────────────────────────┴──────────────────────────────────┘
|
||||
┌─comment──────────────────────────┬─hex(secret)──────────────────────┐
|
||||
│ aes-256-ofb no IV │ B4972BDC4459 │
|
||||
│ aes-256-ofb no IV, different key │ 2FF57C092DC9 │
|
||||
│ aes-256-ofb with IV │ 5E6CB398F653 │
|
||||
│ aes-256-cbc no IV │ 1BC0629A92450D9E73A00E7D02CF4142 │
|
||||
└──────────────────────────────────┴──────────────────────────────────┘
|
||||
```
|
||||
|
||||
Example with `-gcm`:
|
||||
@ -116,9 +115,7 @@ Supported encryption modes:
|
||||
|
||||
- aes-128-ecb, aes-192-ecb, aes-256-ecb
|
||||
- aes-128-cbc, aes-192-cbc, aes-256-cbc
|
||||
- aes-128-cfb1, aes-192-cfb1, aes-256-cfb1
|
||||
- aes-128-cfb8, aes-192-cfb8, aes-256-cfb8
|
||||
- aes-128-cfb128, aes-192-cfb128, aes-256-cfb128
|
||||
- aes-128-cfb128
|
||||
- aes-128-ofb, aes-192-ofb, aes-256-ofb
|
||||
|
||||
**Syntax**
|
||||
@ -145,7 +142,7 @@ Given equal input `encrypt` and `aes_encrypt_mysql` produce the same ciphertext:
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT encrypt('aes-256-cfb128', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv') = aes_encrypt_mysql('aes-256-cfb128', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv') AS ciphertexts_equal;
|
||||
SELECT encrypt('aes-256-ofb', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv') = aes_encrypt_mysql('aes-256-ofb', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv') AS ciphertexts_equal;
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -161,14 +158,14 @@ But `encrypt` fails when `key` or `iv` is longer than expected:
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT encrypt('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123');
|
||||
SELECT encrypt('aes-256-ofb', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
Received exception from server (version 21.1.2):
|
||||
Code: 36. DB::Exception: Received from localhost:9000. DB::Exception: Invalid key size: 33 expected 32: While processing encrypt('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123').
|
||||
Received exception from server (version 22.6.1):
|
||||
Code: 36. DB::Exception: Received from localhost:9000. DB::Exception: Invalid key size: 33 expected 32: While processing encrypt('aes-256-ofb', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123').
|
||||
```
|
||||
|
||||
While `aes_encrypt_mysql` produces MySQL-compatitalbe output:
|
||||
@ -176,7 +173,7 @@ While `aes_encrypt_mysql` produces MySQL-compatitalbe output:
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT hex(aes_encrypt_mysql('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123')) AS ciphertext;
|
||||
SELECT hex(aes_encrypt_mysql('aes-256-ofb', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123')) AS ciphertext;
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -192,7 +189,7 @@ Notice how supplying even longer `IV` produces the same result
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT hex(aes_encrypt_mysql('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456')) AS ciphertext
|
||||
SELECT hex(aes_encrypt_mysql('aes-256-ofb', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456')) AS ciphertext
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -206,7 +203,7 @@ Result:
|
||||
Which is binary equal to what MySQL produces on same inputs:
|
||||
|
||||
``` sql
|
||||
mysql> SET block_encryption_mode='aes-256-cfb128';
|
||||
mysql> SET block_encryption_mode='aes-256-ofb';
|
||||
Query OK, 0 rows affected (0.00 sec)
|
||||
|
||||
mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456') as ciphertext;
|
||||
@ -224,11 +221,10 @@ This function decrypts ciphertext into a plaintext using these modes:
|
||||
|
||||
- aes-128-ecb, aes-192-ecb, aes-256-ecb
|
||||
- aes-128-cbc, aes-192-cbc, aes-256-cbc
|
||||
- aes-128-cfb1, aes-192-cfb1, aes-256-cfb1
|
||||
- aes-128-cfb8, aes-192-cfb8, aes-256-cfb8
|
||||
- aes-128-cfb128, aes-192-cfb128, aes-256-cfb128
|
||||
- aes-128-cfb128
|
||||
- aes-128-ofb, aes-192-ofb, aes-256-ofb
|
||||
- aes-128-gcm, aes-192-gcm, aes-256-gcm
|
||||
- aes-128-ctr, aes-192-ctr, aes-256-ctr
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -265,12 +261,12 @@ Result:
|
||||
│ aes-256-gcm │ A8A3CCBC6426CFEEB60E4EAE03D3E94204C1B09E0254 │
|
||||
│ aes-256-gcm with AAD │ A8A3CCBC6426D9A1017A0A932322F1852260A4AD6837 │
|
||||
└──────────────────────┴──────────────────────────────────────────────┘
|
||||
┌─comment─────────────────────────────┬─hex(secret)──────────────────────┐
|
||||
│ aes-256-cfb128 no IV │ B4972BDC4459 │
|
||||
│ aes-256-cfb128 no IV, different key │ 2FF57C092DC9 │
|
||||
│ aes-256-cfb128 with IV │ 5E6CB398F653 │
|
||||
│ aes-256-cbc no IV │ 1BC0629A92450D9E73A00E7D02CF4142 │
|
||||
└─────────────────────────────────────┴──────────────────────────────────┘
|
||||
┌─comment──────────────────────────┬─hex(secret)──────────────────────┐
|
||||
│ aes-256-ofb no IV │ B4972BDC4459 │
|
||||
│ aes-256-ofb no IV, different key │ 2FF57C092DC9 │
|
||||
│ aes-256-ofb with IV │ 5E6CB398F653 │
|
||||
│ aes-256-cbc no IV │ 1BC0629A92450D9E73A00E7D02CF4142 │
|
||||
└──────────────────────────────────┴──────────────────────────────────┘
|
||||
```
|
||||
|
||||
Now let's try to decrypt all that data.
|
||||
@ -284,13 +280,19 @@ SELECT comment, decrypt('aes-256-cfb128', secret, '12345678910121314151617181920
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─comment─────────────────────────────┬─plaintext─┐
|
||||
│ aes-256-cfb128 no IV │ Secret │
|
||||
│ aes-256-cfb128 no IV, different key │ <20>4<EFBFBD>
|
||||
<20> │
|
||||
│ aes-256-cfb128 with IV │ <20><><EFBFBD>6<EFBFBD>~ │
|
||||
│aes-256-cbc no IV │ <20>2*4<>h3c<33>4w<34><77>@
|
||||
└─────────────────────────────────────┴───────────┘
|
||||
┌─comment──────────────┬─plaintext──┐
|
||||
│ aes-256-gcm │ OQ<4F>E
|
||||
<20>t<EFBFBD>7T<37>\<5C><><EFBFBD>\<5C> │
|
||||
│ aes-256-gcm with AAD │ OQ<4F>E
|
||||
<20>\<5C><>si<73><69><EFBFBD><EFBFBD>;<3B>o<EFBFBD><6F> │
|
||||
└──────────────────────┴────────────┘
|
||||
┌─comment──────────────────────────┬─plaintext─┐
|
||||
│ aes-256-ofb no IV │ Secret │
|
||||
│ aes-256-ofb no IV, different key │ <20>4<EFBFBD>
|
||||
<20> │
|
||||
│ aes-256-ofb with IV │ <20><><EFBFBD>6<EFBFBD>~ │
|
||||
│aes-256-cbc no IV │ <20>2*4<>h3c<33>4w<34><77>@
|
||||
└──────────────────────────────────┴───────────┘
|
||||
```
|
||||
|
||||
Notice how only a portion of the data was properly decrypted, and the rest is gibberish since either `mode`, `key`, or `iv` were different upon encryption.
|
||||
@ -305,9 +307,7 @@ Supported decryption modes:
|
||||
|
||||
- aes-128-ecb, aes-192-ecb, aes-256-ecb
|
||||
- aes-128-cbc, aes-192-cbc, aes-256-cbc
|
||||
- aes-128-cfb1, aes-192-cfb1, aes-256-cfb1
|
||||
- aes-128-cfb8, aes-192-cfb8, aes-256-cfb8
|
||||
- aes-128-cfb128, aes-192-cfb128, aes-256-cfb128
|
||||
- aes-128-cfb128
|
||||
- aes-128-ofb, aes-192-ofb, aes-256-ofb
|
||||
|
||||
**Syntax**
|
||||
@ -332,7 +332,7 @@ aes_decrypt_mysql('mode', 'ciphertext', 'key' [, iv])
|
||||
Let's decrypt data we've previously encrypted with MySQL:
|
||||
|
||||
``` sql
|
||||
mysql> SET block_encryption_mode='aes-256-cfb128';
|
||||
mysql> SET block_encryption_mode='aes-256-ofb';
|
||||
Query OK, 0 rows affected (0.00 sec)
|
||||
|
||||
mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456') as ciphertext;
|
||||
@ -347,7 +347,7 @@ mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviv
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT aes_decrypt_mysql('aes-256-cfb128', unhex('24E9E4966469'), '123456789101213141516171819202122', 'iviviviviviviviv123456') AS plaintext
|
||||
SELECT aes_decrypt_mysql('aes-256-ofb', unhex('24E9E4966469'), '123456789101213141516171819202122', 'iviviviviviviviv123456') AS plaintext
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
@ -273,16 +273,16 @@ Converts ASCII Latin symbols in a string to uppercase.
|
||||
## lowerUTF8
|
||||
|
||||
Converts a string to lowercase, assuming the string contains a set of bytes that make up a UTF-8 encoded text.
|
||||
It does not detect the language. So for Turkish the result might not be exactly correct.
|
||||
It does not detect the language. E.g. for Turkish the result might not be exactly correct (i/İ vs. i/I).
|
||||
If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point.
|
||||
If the string contains a set of bytes that is not UTF-8, then the behavior is undefined.
|
||||
If the string contains a sequence of bytes that are not valid UTF-8, then the behavior is undefined.
|
||||
|
||||
## upperUTF8
|
||||
|
||||
Converts a string to uppercase, assuming the string contains a set of bytes that make up a UTF-8 encoded text.
|
||||
It does not detect the language. So for Turkish the result might not be exactly correct.
|
||||
It does not detect the language. E.g. for Turkish the result might not be exactly correct (i/İ vs. i/I).
|
||||
If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point.
|
||||
If the string contains a set of bytes that is not UTF-8, then the behavior is undefined.
|
||||
If the string contains a sequence of bytes that are not valid UTF-8, then the behavior is undefined.
|
||||
|
||||
## isValidUTF8
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: For Searching in Strings
|
||||
|
||||
The search is case-sensitive by default in all these functions. There are separate variants for case insensitive search.
|
||||
|
||||
:::note
|
||||
:::note
|
||||
Functions for [replacing](../../sql-reference/functions/string-replace-functions.md) and [other manipulations with strings](../../sql-reference/functions/string-functions.md) are described separately.
|
||||
:::
|
||||
|
||||
@ -31,7 +31,7 @@ position(needle IN haystack)
|
||||
|
||||
Alias: `locate(haystack, needle[, start_pos])`.
|
||||
|
||||
:::note
|
||||
:::note
|
||||
Syntax of `position(needle IN haystack)` provides SQL-compatibility, the function works the same way as to `position(haystack, needle)`.
|
||||
:::
|
||||
|
||||
@ -344,24 +344,27 @@ Returns 1, if at least one string needle<sub>i</sub> matches the string `haystac
|
||||
|
||||
For a case-insensitive search or/and in UTF-8 format use functions `multiSearchAnyCaseInsensitive, multiSearchAnyUTF8, multiSearchAnyCaseInsensitiveUTF8`.
|
||||
|
||||
:::note
|
||||
:::note
|
||||
In all `multiSearch*` functions the number of needles should be less than 2<sup>8</sup> because of implementation specification.
|
||||
:::
|
||||
|
||||
## match(haystack, pattern)
|
||||
|
||||
Checks whether the string matches the `pattern` regular expression. A `re2` regular expression. The [syntax](https://github.com/google/re2/wiki/Syntax) of the `re2` regular expressions is more limited than the syntax of the Perl regular expressions.
|
||||
Checks whether the string matches the regular expression `pattern` in `re2` syntax. `Re2` has a more limited [syntax](https://github.com/google/re2/wiki/Syntax) than Perl regular expressions.
|
||||
|
||||
Returns 0 if it does not match, or 1 if it matches.
|
||||
|
||||
The regular expression works with the string as if it is a set of bytes. The regular expression can’t contain null bytes.
|
||||
Matching is based on UTF-8, e.g. `.` matches the Unicode code point `¥` which is represented in UTF-8 using two bytes. The regular expression must not contain null bytes.
|
||||
If the haystack or pattern contain a sequence of bytes that are not valid UTF-8, then the behavior is undefined.
|
||||
No automatic Unicode normalization is performed, if you need it you can use the [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that.
|
||||
|
||||
For patterns to search for substrings in a string, it is better to use LIKE or ‘position’, since they work much faster.
|
||||
|
||||
## multiMatchAny(haystack, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\])
|
||||
|
||||
The same as `match`, but returns 0 if none of the regular expressions are matched and 1 if any of the patterns matches. It uses [hyperscan](https://github.com/intel/hyperscan) library. For patterns to search substrings in a string, it is better to use `multiSearchAny` since it works much faster.
|
||||
|
||||
:::note
|
||||
:::note
|
||||
The length of any of the `haystack` string must be less than 2<sup>32</sup> bytes otherwise the exception is thrown. This restriction takes place because of hyperscan API.
|
||||
:::
|
||||
|
||||
@ -385,11 +388,11 @@ The same as `multiFuzzyMatchAny`, but returns any index that matches the haystac
|
||||
|
||||
The same as `multiFuzzyMatchAny`, but returns the array of all indices in any order that match the haystack within a constant edit distance.
|
||||
|
||||
:::note
|
||||
:::note
|
||||
`multiFuzzyMatch*` functions do not support UTF-8 regular expressions, and such expressions are treated as bytes because of hyperscan restriction.
|
||||
:::
|
||||
|
||||
:::note
|
||||
:::note
|
||||
To turn off all functions that use hyperscan, use setting `SET allow_hyperscan = 0;`.
|
||||
:::
|
||||
|
||||
@ -405,7 +408,7 @@ Extracts all the fragments of a string using a regular expression. If ‘haystac
|
||||
|
||||
Matches all groups of the `haystack` string using the `pattern` regular expression. Returns an array of arrays, where the first array includes all fragments matching the first group, the second array - matching the second group, etc.
|
||||
|
||||
:::note
|
||||
:::note
|
||||
`extractAllGroupsHorizontal` function is slower than [extractAllGroupsVertical](#extractallgroups-vertical).
|
||||
:::
|
||||
|
||||
@ -498,6 +501,10 @@ The regular expression can contain the metasymbols `%` and `_`.
|
||||
|
||||
Use the backslash (`\`) for escaping metasymbols. See the note on escaping in the description of the ‘match’ function.
|
||||
|
||||
Matching is based on UTF-8, e.g. `_` matches the Unicode code point `¥` which is represented in UTF-8 using two bytes.
|
||||
If the haystack or pattern contain a sequence of bytes that are not valid UTF-8, then the behavior is undefined.
|
||||
No automatic Unicode normalization is performed, if you need it you can use the [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that.
|
||||
|
||||
For regular expressions like `%needle%`, the code is more optimal and works as fast as the `position` function.
|
||||
For other regular expressions, the code is the same as for the ‘match’ function.
|
||||
|
||||
@ -509,6 +516,8 @@ The same thing as ‘like’, but negative.
|
||||
|
||||
Case insensitive variant of [like](https://clickhouse.com/docs/en/sql-reference/functions/string-search-functions/#function-like) function. You can use `ILIKE` operator instead of the `ilike` function.
|
||||
|
||||
The function ignores the language, e.g. for Turkish (i/İ), the result might be incorrect.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
@ -577,7 +586,7 @@ Same as `ngramDistance` but calculates the non-symmetric difference between `nee
|
||||
|
||||
For case-insensitive search or/and in UTF-8 format use functions `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8`.
|
||||
|
||||
:::note
|
||||
:::note
|
||||
For UTF-8 case we use 3-gram distance. All these are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the (non-)symmetric difference between these hash tables – collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function – we zero the 5-th bit (starting from zero) of each codepoint byte and first bit of zeroth byte if bytes more than one – this works for Latin and mostly for all Cyrillic letters.
|
||||
:::
|
||||
|
||||
|
||||
@ -43,28 +43,38 @@ For tuple subtraction: [tupleMinus](../../sql-reference/functions/tuple-function
|
||||
|
||||
## Comparison Operators
|
||||
|
||||
### equals function
|
||||
`a = b` – The `equals(a, b)` function.
|
||||
|
||||
`a == b` – The `equals(a, b)` function.
|
||||
|
||||
### notEquals function
|
||||
`a != b` – The `notEquals(a, b)` function.
|
||||
|
||||
`a <> b` – The `notEquals(a, b)` function.
|
||||
|
||||
### lessOrEquals function
|
||||
`a <= b` – The `lessOrEquals(a, b)` function.
|
||||
|
||||
### greaterOrEquals function
|
||||
`a >= b` – The `greaterOrEquals(a, b)` function.
|
||||
|
||||
### less function
|
||||
`a < b` – The `less(a, b)` function.
|
||||
|
||||
### greater function
|
||||
`a > b` – The `greater(a, b)` function.
|
||||
|
||||
### like function
|
||||
`a LIKE s` – The `like(a, b)` function.
|
||||
|
||||
### notLike function
|
||||
`a NOT LIKE s` – The `notLike(a, b)` function.
|
||||
|
||||
### ilike function
|
||||
`a ILIKE s` – The `ilike(a, b)` function.
|
||||
|
||||
### BETWEEN function
|
||||
`a BETWEEN b AND c` – The same as `a >= b AND a <= c`.
|
||||
|
||||
`a NOT BETWEEN b AND c` – The same as `a < b OR a > c`.
|
||||
@ -73,20 +83,28 @@ For tuple subtraction: [tupleMinus](../../sql-reference/functions/tuple-function
|
||||
|
||||
See [IN operators](../../sql-reference/operators/in.md) and [EXISTS](../../sql-reference/operators/exists.md) operator.
|
||||
|
||||
### in function
|
||||
`a IN ...` – The `in(a, b)` function.
|
||||
|
||||
### notIn function
|
||||
`a NOT IN ...` – The `notIn(a, b)` function.
|
||||
|
||||
### globalIn function
|
||||
`a GLOBAL IN ...` – The `globalIn(a, b)` function.
|
||||
|
||||
### globalNotIn function
|
||||
`a GLOBAL NOT IN ...` – The `globalNotIn(a, b)` function.
|
||||
|
||||
### in subquery function
|
||||
`a = ANY (subquery)` – The `in(a, subquery)` function.
|
||||
|
||||
### notIn subquery function
|
||||
`a != ANY (subquery)` – The same as `a NOT IN (SELECT singleValueOrNull(*) FROM subquery)`.
|
||||
|
||||
### in subquery function
|
||||
`a = ALL (subquery)` – The same as `a IN (SELECT singleValueOrNull(*) FROM subquery)`.
|
||||
|
||||
### notIn subquery function
|
||||
`a != ALL (subquery)` – The `notIn(a, subquery)` function.
|
||||
|
||||
|
||||
|
||||
@ -48,9 +48,9 @@ You can see that `GROUP BY` for `y = NULL` summed up `x`, as if `NULL` is this v
|
||||
|
||||
If you pass several keys to `GROUP BY`, the result will give you all the combinations of the selection, as if `NULL` were a specific value.
|
||||
|
||||
## WITH ROLLUP Modifier
|
||||
## ROLLUP Modifier
|
||||
|
||||
`WITH ROLLUP` modifier is used to calculate subtotals for the key expressions, based on their order in the `GROUP BY` list. The subtotals rows are added after the result table.
|
||||
`ROLLUP` modifier is used to calculate subtotals for the key expressions, based on their order in the `GROUP BY` list. The subtotals rows are added after the result table.
|
||||
|
||||
The subtotals are calculated in the reverse order: at first subtotals are calculated for the last key expression in the list, then for the previous one, and so on up to the first key expression.
|
||||
|
||||
@ -78,7 +78,7 @@ Consider the table t:
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT year, month, day, count(*) FROM t GROUP BY year, month, day WITH ROLLUP;
|
||||
SELECT year, month, day, count(*) FROM t GROUP BY ROLLUP(year, month, day);
|
||||
```
|
||||
As `GROUP BY` section has three key expressions, the result contains four tables with subtotals "rolled up" from right to left:
|
||||
|
||||
@ -109,10 +109,14 @@ As `GROUP BY` section has three key expressions, the result contains four tables
|
||||
│ 0 │ 0 │ 0 │ 6 │
|
||||
└──────┴───────┴─────┴─────────┘
|
||||
```
|
||||
The same query also can be written using `WITH` keyword.
|
||||
```sql
|
||||
SELECT year, month, day, count(*) FROM t GROUP BY year, month, day WITH ROLLUP;
|
||||
```
|
||||
|
||||
## WITH CUBE Modifier
|
||||
## CUBE Modifier
|
||||
|
||||
`WITH CUBE` modifier is used to calculate subtotals for every combination of the key expressions in the `GROUP BY` list. The subtotals rows are added after the result table.
|
||||
`CUBE` modifier is used to calculate subtotals for every combination of the key expressions in the `GROUP BY` list. The subtotals rows are added after the result table.
|
||||
|
||||
In the subtotals rows the values of all "grouped" key expressions are set to `0` or empty line.
|
||||
|
||||
@ -138,7 +142,7 @@ Consider the table t:
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT year, month, day, count(*) FROM t GROUP BY year, month, day WITH CUBE;
|
||||
SELECT year, month, day, count(*) FROM t GROUP BY CUBE(year, month, day);
|
||||
```
|
||||
|
||||
As `GROUP BY` section has three key expressions, the result contains eight tables with subtotals for all key expression combinations:
|
||||
@ -196,6 +200,10 @@ Columns, excluded from `GROUP BY`, are filled with zeros.
|
||||
│ 0 │ 0 │ 0 │ 6 │
|
||||
└──────┴───────┴─────┴─────────┘
|
||||
```
|
||||
The same query also can be written using `WITH` keyword.
|
||||
```sql
|
||||
SELECT year, month, day, count(*) FROM t GROUP BY year, month, day WITH CUBE;
|
||||
```
|
||||
|
||||
|
||||
## WITH TOTALS Modifier
|
||||
@ -260,6 +268,39 @@ GROUP BY domain
|
||||
|
||||
For every different key value encountered, `GROUP BY` calculates a set of aggregate function values.
|
||||
|
||||
## GROUPING SETS modifier
|
||||
|
||||
This is the most general modifier.
|
||||
This modifier allows to manually specify several aggregation key sets (grouping sets).
|
||||
Aggregation is performed separately for each grouping set, after that all results are combined.
|
||||
If a column is not presented in a grouping set, it's filled with a default value.
|
||||
|
||||
In other words, modifiers described above can be represented via `GROUPING SETS`.
|
||||
Despite the fact that queries with `ROLLUP`, `CUBE` and `GROUPING SETS` modifiers are syntactically equal, they may have different performance.
|
||||
When `GROUPING SETS` try to execute everything in parallel, `ROLLUP` and `CUBE` are executing the final merging of the aggregates in a single thread.
|
||||
|
||||
In the situation when source columns contain default values, it might be hard to distinguish if a row is a part of the aggregation which uses those columns as keys or not.
|
||||
To solve this problem `GROUPING` function must be used.
|
||||
|
||||
**Example**
|
||||
|
||||
The following two queries are equivalent.
|
||||
|
||||
```sql
|
||||
-- Query 1
|
||||
SELECT year, month, day, count(*) FROM t GROUP BY year, month, day WITH ROLLUP;
|
||||
|
||||
-- Query 2
|
||||
SELECT year, month, day, count(*) FROM t GROUP BY
|
||||
GROUPING SETS
|
||||
(
|
||||
(year, month, day),
|
||||
(year, month),
|
||||
(year),
|
||||
()
|
||||
);
|
||||
```
|
||||
|
||||
## Implementation Details
|
||||
|
||||
Aggregation is one of the most important features of a column-oriented DBMS, and thus it’s implementation is one of the most heavily optimized parts of ClickHouse. By default, aggregation is done in memory using a hash-table. It has 40+ specializations that are chosen automatically depending on “grouping key” data types.
|
||||
|
||||
@ -10,7 +10,7 @@ ClickHouse supports the standard grammar for defining windows and window functio
|
||||
| Feature | Support or workaround |
|
||||
| --------| ----------|
|
||||
| ad hoc window specification (`count(*) over (partition by id order by time desc)`) | supported |
|
||||
| expressions involving window functions, e.g. `(count(*) over ()) / 2)` | not supported, wrap in a subquery ([feature request](https://github.com/ClickHouse/ClickHouse/issues/19857)) |
|
||||
| expressions involving window functions, e.g. `(count(*) over ()) / 2)` | supported |
|
||||
| `WINDOW` clause (`select ... from table window w as (partition by id)`) | supported |
|
||||
| `ROWS` frame | supported |
|
||||
| `RANGE` frame | supported, the default |
|
||||
@ -55,3 +55,372 @@ https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html
|
||||
https://dev.mysql.com/doc/refman/8.0/en/window-functions-usage.html
|
||||
|
||||
https://dev.mysql.com/doc/refman/8.0/en/window-functions-frames.html
|
||||
|
||||
## Syntax
|
||||
|
||||
```text
|
||||
aggregate_function (column_name)
|
||||
OVER ([PARTITION BY groupping_column] [ORDER BY sorting_column]
|
||||
[ROWS or RANGE expression_to_bounds_of_frame])
|
||||
```
|
||||
|
||||
- `PARTITION BY` - defines how to break a resultset into groups.
|
||||
- `ORDER BY` - defines how to order rows inside the group during calculation aggregate_function.
|
||||
- `ROWS or RANGE` - defines bounds of a frame, aggregate_function is calculated within a frame.
|
||||
|
||||
```text
|
||||
PARTITION
|
||||
┌─────────────────┐ <-- UNBOUNDED PRECEDING (BEGINNING of the PARTITION)
|
||||
│ │
|
||||
│ │
|
||||
│=================│ <-- N PRECEDING <─┐
|
||||
│ N ROWS │ │ F
|
||||
│ Before CURRENT │ │ R
|
||||
│~~~~~~~~~~~~~~~~~│ <-- CURRENT ROW │ A
|
||||
│ M ROWS │ │ M
|
||||
│ After CURRENT │ │ E
|
||||
│=================│ <-- M FOLLOWING <─┘
|
||||
│ │
|
||||
│ │
|
||||
└─────────────────┘ <--- UNBOUNDED FOLLOWING (END of the PARTITION)
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
```sql
|
||||
CREATE TABLE wf_partition
|
||||
(
|
||||
`part_key` UInt64,
|
||||
`value` UInt64
|
||||
)
|
||||
ENGINE = Memory;
|
||||
|
||||
INSERT INTO wf_partition FORMAT Values
|
||||
(1,1,1), (1,2,2), (1,3,3), (2,0,0), (3,0,0);
|
||||
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key) AS frame_values
|
||||
FROM wf_partition
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
value ASC;
|
||||
|
||||
┌─part_key─┬─value─┬─order─┬─frame_values─┐
|
||||
│ 1 │ 1 │ 1 │ [1,2,3] │ <┐
|
||||
│ 1 │ 2 │ 2 │ [1,2,3] │ │ 1-st group
|
||||
│ 1 │ 3 │ 3 │ [1,2,3] │ <┘
|
||||
│ 2 │ 0 │ 0 │ [0] │ <- 2-nd group
|
||||
│ 3 │ 0 │ 0 │ [0] │ <- 3-d group
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE wf_frame
|
||||
(
|
||||
`part_key` UInt64,
|
||||
`value` UInt64,
|
||||
`order` UInt64
|
||||
)
|
||||
ENGINE = Memory;
|
||||
|
||||
INSERT INTO wf_frame FORMAT Values
|
||||
(1,1,1), (1,2,2), (1,3,3), (1,4,4), (1,5,5);
|
||||
|
||||
-- frame is bounded by bounds of a partition (BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
|
||||
Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
value ASC;
|
||||
|
||||
┌─part_key─┬─value─┬─order─┬─frame_values─┐
|
||||
│ 1 │ 1 │ 1 │ [1,2,3,4,5] │
|
||||
│ 1 │ 2 │ 2 │ [1,2,3,4,5] │
|
||||
│ 1 │ 3 │ 3 │ [1,2,3,4,5] │
|
||||
│ 1 │ 4 │ 4 │ [1,2,3,4,5] │
|
||||
│ 1 │ 5 │ 5 │ [1,2,3,4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
|
||||
-- short form - no bound expression, no order by
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
value ASC;
|
||||
┌─part_key─┬─value─┬─order─┬─frame_values─┐
|
||||
│ 1 │ 1 │ 1 │ [1,2,3,4,5] │
|
||||
│ 1 │ 2 │ 2 │ [1,2,3,4,5] │
|
||||
│ 1 │ 3 │ 3 │ [1,2,3,4,5] │
|
||||
│ 1 │ 4 │ 4 │ [1,2,3,4,5] │
|
||||
│ 1 │ 5 │ 5 │ [1,2,3,4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
|
||||
-- frame is bounded by the beggining of a partition and the current row
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
|
||||
Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
value ASC;
|
||||
|
||||
┌─part_key─┬─value─┬─order─┬─frame_values─┐
|
||||
│ 1 │ 1 │ 1 │ [1] │
|
||||
│ 1 │ 2 │ 2 │ [1,2] │
|
||||
│ 1 │ 3 │ 3 │ [1,2,3] │
|
||||
│ 1 │ 4 │ 4 │ [1,2,3,4] │
|
||||
│ 1 │ 5 │ 5 │ [1,2,3,4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
|
||||
-- short form (frame is bounded by the beggining of a partition and the current row)
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
value ASC;
|
||||
┌─part_key─┬─value─┬─order─┬─frame_values─┐
|
||||
│ 1 │ 1 │ 1 │ [1] │
|
||||
│ 1 │ 2 │ 2 │ [1,2] │
|
||||
│ 1 │ 3 │ 3 │ [1,2,3] │
|
||||
│ 1 │ 4 │ 4 │ [1,2,3,4] │
|
||||
│ 1 │ 5 │ 5 │ [1,2,3,4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
|
||||
-- frame is bounded by the beggining of a partition and the current row, but order is backward
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key ORDER BY order DESC) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
value ASC;
|
||||
┌─part_key─┬─value─┬─order─┬─frame_values─┐
|
||||
│ 1 │ 1 │ 1 │ [5,4,3,2,1] │
|
||||
│ 1 │ 2 │ 2 │ [5,4,3,2] │
|
||||
│ 1 │ 3 │ 3 │ [5,4,3] │
|
||||
│ 1 │ 4 │ 4 │ [5,4] │
|
||||
│ 1 │ 5 │ 5 │ [5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
|
||||
-- sliding frame - 1 PRECEDING ROW AND CURRENT ROW
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
|
||||
Rows BETWEEN 1 PRECEDING AND CURRENT ROW) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
value ASC;
|
||||
|
||||
┌─part_key─┬─value─┬─order─┬─frame_values─┐
|
||||
│ 1 │ 1 │ 1 │ [1] │
|
||||
│ 1 │ 2 │ 2 │ [1,2] │
|
||||
│ 1 │ 3 │ 3 │ [2,3] │
|
||||
│ 1 │ 4 │ 4 │ [3,4] │
|
||||
│ 1 │ 5 │ 5 │ [4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
|
||||
-- sliding frame - Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
|
||||
Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
value ASC;
|
||||
┌─part_key─┬─value─┬─order─┬─frame_values─┐
|
||||
│ 1 │ 1 │ 1 │ [1,2,3,4,5] │
|
||||
│ 1 │ 2 │ 2 │ [1,2,3,4,5] │
|
||||
│ 1 │ 3 │ 3 │ [2,3,4,5] │
|
||||
│ 1 │ 4 │ 4 │ [3,4,5] │
|
||||
│ 1 │ 5 │ 5 │ [4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
```
|
||||
|
||||
## Real world examples
|
||||
|
||||
### Maximum/total salary per department.
|
||||
|
||||
```sql
|
||||
CREATE TABLE employees
|
||||
(
|
||||
`department` String,
|
||||
`employee_name` String,
|
||||
`salary` Float
|
||||
)
|
||||
ENGINE = Memory;
|
||||
|
||||
INSERT INTO employees FORMAT Values
|
||||
('Finance', 'Jonh', 200),
|
||||
('Finance', 'Joan', 210),
|
||||
('Finance', 'Jean', 505),
|
||||
('IT', 'Tim', 200),
|
||||
('IT', 'Anna', 300),
|
||||
('IT', 'Elen', 500);
|
||||
|
||||
SELECT
|
||||
department,
|
||||
employee_name AS emp,
|
||||
salary,
|
||||
max_salary_per_dep,
|
||||
total_salary_per_dep,
|
||||
round((salary / total_salary_per_dep) * 100, 2) AS `share_per_dep(%)`
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
department,
|
||||
employee_name,
|
||||
salary,
|
||||
max(salary) OVER wndw AS max_salary_per_dep,
|
||||
sum(salary) OVER wndw AS total_salary_per_dep
|
||||
FROM employees
|
||||
WINDOW wndw AS (PARTITION BY department
|
||||
rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
|
||||
ORDER BY
|
||||
department ASC,
|
||||
employee_name ASC
|
||||
);
|
||||
|
||||
┌─department─┬─emp──┬─salary─┬─max_salary_per_dep─┬─total_salary_per_dep─┬─share_per_dep(%)─┐
|
||||
│ Finance │ Jean │ 505 │ 505 │ 915 │ 55.19 │
|
||||
│ Finance │ Joan │ 210 │ 505 │ 915 │ 22.95 │
|
||||
│ Finance │ Jonh │ 200 │ 505 │ 915 │ 21.86 │
|
||||
│ IT │ Anna │ 300 │ 500 │ 1000 │ 30 │
|
||||
│ IT │ Elen │ 500 │ 500 │ 1000 │ 50 │
|
||||
│ IT │ Tim │ 200 │ 500 │ 1000 │ 20 │
|
||||
└────────────┴──────┴────────┴────────────────────┴──────────────────────┴──────────────────┘
|
||||
```
|
||||
|
||||
### Cumulative sum.
|
||||
|
||||
```sql
|
||||
CREATE TABLE events
|
||||
(
|
||||
`metric` String,
|
||||
`ts` DateTime,
|
||||
`value` Float
|
||||
)
|
||||
ENGINE = Memory
|
||||
|
||||
INSERT INTO warehouse VALUES
|
||||
('sku38', '2020-01-01', 9),
|
||||
('sku38', '2020-02-01', 1),
|
||||
('sku38', '2020-03-01', -4),
|
||||
('sku1', '2020-01-01', 1),
|
||||
('sku1', '2020-02-01', 1),
|
||||
('sku1', '2020-03-01', 1);
|
||||
|
||||
SELECT
|
||||
item,
|
||||
ts,
|
||||
value,
|
||||
sum(value) OVER (PARTITION BY item ORDER BY ts ASC) AS stock_balance
|
||||
FROM warehouse
|
||||
ORDER BY
|
||||
item ASC,
|
||||
ts ASC;
|
||||
|
||||
┌─item──┬──────────────────ts─┬─value─┬─stock_balance─┐
|
||||
│ sku1 │ 2020-01-01 00:00:00 │ 1 │ 1 │
|
||||
│ sku1 │ 2020-02-01 00:00:00 │ 1 │ 2 │
|
||||
│ sku1 │ 2020-03-01 00:00:00 │ 1 │ 3 │
|
||||
│ sku38 │ 2020-01-01 00:00:00 │ 9 │ 9 │
|
||||
│ sku38 │ 2020-02-01 00:00:00 │ 1 │ 10 │
|
||||
│ sku38 │ 2020-03-01 00:00:00 │ -4 │ 6 │
|
||||
└───────┴─────────────────────┴───────┴───────────────┘
|
||||
```
|
||||
|
||||
### Moving / Sliding Average (per 3 rows)
|
||||
|
||||
```sql
|
||||
CREATE TABLE sensors
|
||||
(
|
||||
`metric` String,
|
||||
`ts` DateTime,
|
||||
`value` Float
|
||||
)
|
||||
ENGINE = Memory;
|
||||
|
||||
insert into sensors values('cpu_temp', '2020-01-01 00:00:00', 87),
|
||||
('cpu_temp', '2020-01-01 00:00:01', 77),
|
||||
('cpu_temp', '2020-01-01 00:00:02', 93),
|
||||
('cpu_temp', '2020-01-01 00:00:03', 87),
|
||||
('cpu_temp', '2020-01-01 00:00:04', 87),
|
||||
('cpu_temp', '2020-01-01 00:00:05', 87),
|
||||
('cpu_temp', '2020-01-01 00:00:06', 87),
|
||||
('cpu_temp', '2020-01-01 00:00:07', 87);
|
||||
SELECT
|
||||
metric,
|
||||
ts,
|
||||
value,
|
||||
avg(value) OVER
|
||||
(PARTITION BY metric ORDER BY ts ASC Rows BETWEEN 2 PRECEDING AND CURRENT ROW)
|
||||
AS moving_avg_temp
|
||||
FROM sensors
|
||||
ORDER BY
|
||||
metric ASC,
|
||||
ts ASC;
|
||||
|
||||
┌─metric───┬──────────────────ts─┬─value─┬───moving_avg_temp─┐
|
||||
│ cpu_temp │ 2020-01-01 00:00:00 │ 87 │ 87 │
|
||||
│ cpu_temp │ 2020-01-01 00:00:01 │ 77 │ 82 │
|
||||
│ cpu_temp │ 2020-01-01 00:00:02 │ 93 │ 85.66666666666667 │
|
||||
│ cpu_temp │ 2020-01-01 00:00:03 │ 87 │ 85.66666666666667 │
|
||||
│ cpu_temp │ 2020-01-01 00:00:04 │ 87 │ 89 │
|
||||
│ cpu_temp │ 2020-01-01 00:00:05 │ 87 │ 87 │
|
||||
│ cpu_temp │ 2020-01-01 00:00:06 │ 87 │ 87 │
|
||||
│ cpu_temp │ 2020-01-01 00:00:07 │ 87 │ 87 │
|
||||
└──────────┴─────────────────────┴───────┴───────────────────┘
|
||||
```
|
||||
|
||||
### Moving / Sliding Average (per 10 seconds)
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
metric,
|
||||
ts,
|
||||
value,
|
||||
avg(value) OVER (PARTITION BY metric ORDER BY ts
|
||||
Range BETWEEN 10 PRECEDING AND CURRENT ROW) AS moving_avg_10_seconds_temp
|
||||
FROM sensors
|
||||
ORDER BY
|
||||
metric ASC,
|
||||
ts ASC;
|
||||
|
||||
┌─metric───┬──────────────────ts─┬─value─┬─moving_avg_10_seconds_temp─┐
|
||||
│ cpu_temp │ 2020-01-01 00:00:00 │ 87 │ 87 │
|
||||
│ cpu_temp │ 2020-01-01 00:01:10 │ 77 │ 77 │
|
||||
│ cpu_temp │ 2020-01-01 00:02:20 │ 93 │ 93 │
|
||||
│ cpu_temp │ 2020-01-01 00:03:30 │ 87 │ 87 │
|
||||
│ cpu_temp │ 2020-01-01 00:04:40 │ 87 │ 87 │
|
||||
│ cpu_temp │ 2020-01-01 00:05:50 │ 87 │ 87 │
|
||||
│ cpu_temp │ 2020-01-01 00:06:00 │ 87 │ 87 │
|
||||
│ cpu_temp │ 2020-01-01 00:07:10 │ 87 │ 87 │
|
||||
└──────────┴─────────────────────┴───────┴────────────────────────────┘
|
||||
```
|
||||
|
||||
@ -39,6 +39,6 @@ Question candidates:
|
||||
- How to kill a process (query) in ClickHouse?
|
||||
- How to implement pivot (like in pandas)?
|
||||
- How to remove the default ClickHouse user through users.d?
|
||||
- Importing MySQL dump to Clickhouse
|
||||
- Importing MySQL dump to ClickHouse
|
||||
- Window function workarounds (row\_number, lag/lead, running diff/sum/average)
|
||||
##}
|
||||
|
||||
@ -121,7 +121,7 @@ $ clickhouse-client --param_tbl="numbers" --param_db="system" --param_col="numbe
|
||||
- `--user, -u` — имя пользователя, по умолчанию — ‘default’.
|
||||
- `--password` — пароль, по умолчанию — пустая строка.
|
||||
- `--query, -q` — запрос для выполнения, при использовании в неинтерактивном режиме.
|
||||
- `--queries-file, -qf` - путь к файлу с запросами для выполнения. Необходимо указать только одну из опций: `query` или `queries-file`.
|
||||
- `--queries-file` - путь к файлу с запросами для выполнения. Необходимо указать только одну из опций: `query` или `queries-file`.
|
||||
- `--database, -d` — выбрать текущую БД. Без указания значение берется из настроек сервера (по умолчанию — БД ‘default’).
|
||||
- `--multiline, -m` — если указано — разрешить многострочные запросы, не отправлять запрос по нажатию Enter.
|
||||
- `--multiquery, -n` — если указано — разрешить выполнять несколько запросов, разделённых точкой с запятой.
|
||||
|
||||
@ -28,12 +28,12 @@ $ clickhouse-local --structure "table_structure" --input-format "format_of_incom
|
||||
Ключи команды:
|
||||
|
||||
- `-S`, `--structure` — структура таблицы, в которую будут помещены входящие данные.
|
||||
- `-if`, `--input-format` — формат входящих данных. По умолчанию — `TSV`.
|
||||
- `--input-format` — формат входящих данных. По умолчанию — `TSV`.
|
||||
- `-f`, `--file` — путь к файлу с данными. По умолчанию — `stdin`.
|
||||
- `-q`, `--query` — запросы на выполнение. Разделитель запросов — `;`.
|
||||
- `-qf`, `--queries-file` - путь к файлу с запросами для выполнения. Необходимо задать либо параметр `query`, либо `queries-file`.
|
||||
- `--queries-file` - путь к файлу с запросами для выполнения. Необходимо задать либо параметр `query`, либо `queries-file`.
|
||||
- `-N`, `--table` — имя таблицы, в которую будут помещены входящие данные. По умолчанию - `table`.
|
||||
- `-of`, `--format`, `--output-format` — формат выходных данных. По умолчанию — `TSV`.
|
||||
- `--format`, `--output-format` — формат выходных данных. По умолчанию — `TSV`.
|
||||
- `-d`, `--database` — база данных по умолчанию. Если не указано, используется значение `_local`.
|
||||
- `--stacktrace` — вывод отладочной информации при исключениях.
|
||||
- `--echo` — перед выполнением запрос выводится в консоль.
|
||||
@ -109,4 +109,3 @@ Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec.
|
||||
├──────────┼──────────┤
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
@ -55,5 +55,5 @@ ORDER BY id
|
||||
|
||||
## Смотрите также
|
||||
|
||||
- [Reducing Clickhouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/).
|
||||
- [Reducing ClickHouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/).
|
||||
- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/ClickHouse/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf).
|
||||
|
||||
@ -11,7 +11,7 @@ sidebar_label: "Функции для шифрования"
|
||||
|
||||
Длина инициализирующего вектора всегда 16 байт (лишние байты игнорируются).
|
||||
|
||||
Обратите внимание, что до версии Clickhouse 21.1 эти функции работали медленно.
|
||||
Обратите внимание, что до версии ClickHouse 21.1 эти функции работали медленно.
|
||||
|
||||
## encrypt {#encrypt}
|
||||
|
||||
@ -19,11 +19,10 @@ sidebar_label: "Функции для шифрования"
|
||||
|
||||
- aes-128-ecb, aes-192-ecb, aes-256-ecb
|
||||
- aes-128-cbc, aes-192-cbc, aes-256-cbc
|
||||
- aes-128-cfb1, aes-192-cfb1, aes-256-cfb1
|
||||
- aes-128-cfb8, aes-192-cfb8, aes-256-cfb8
|
||||
- aes-128-cfb128, aes-192-cfb128, aes-256-cfb128
|
||||
- aes-128-cfb128
|
||||
- aes-128-ofb, aes-192-ofb, aes-256-ofb
|
||||
- aes-128-gcm, aes-192-gcm, aes-256-gcm
|
||||
- aes-128-ctr, aes-192-ctr, aes-256-ctr
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
@ -63,9 +62,9 @@ ENGINE = Memory;
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
INSERT INTO encryption_test VALUES('aes-256-cfb128 no IV', encrypt('aes-256-cfb128', 'Secret', '12345678910121314151617181920212')),\
|
||||
('aes-256-cfb128 no IV, different key', encrypt('aes-256-cfb128', 'Secret', 'keykeykeykeykeykeykeykeykeykeyke')),\
|
||||
('aes-256-cfb128 with IV', encrypt('aes-256-cfb128', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv')),\
|
||||
INSERT INTO encryption_test VALUES('aes-256-ofb no IV', encrypt('aes-256-ofb', 'Secret', '12345678910121314151617181920212')),\
|
||||
('aes-256-ofb no IV, different key', encrypt('aes-256-ofb', 'Secret', 'keykeykeykeykeykeykeykeykeykeyke')),\
|
||||
('aes-256-ofb with IV', encrypt('aes-256-ofb', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv')),\
|
||||
('aes-256-cbc no IV', encrypt('aes-256-cbc', 'Secret', '12345678910121314151617181920212'));
|
||||
```
|
||||
|
||||
@ -78,12 +77,12 @@ SELECT comment, hex(secret) FROM encryption_test;
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─comment─────────────────────────────┬─hex(secret)──────────────────────┐
|
||||
│ aes-256-cfb128 no IV │ B4972BDC4459 │
|
||||
│ aes-256-cfb128 no IV, different key │ 2FF57C092DC9 │
|
||||
│ aes-256-cfb128 with IV │ 5E6CB398F653 │
|
||||
│ aes-256-cbc no IV │ 1BC0629A92450D9E73A00E7D02CF4142 │
|
||||
└─────────────────────────────────────┴──────────────────────────────────┘
|
||||
┌─comment──────────────────────────┬─hex(secret)──────────────────────┐
|
||||
│ aes-256-ofb no IV │ B4972BDC4459 │
|
||||
│ aes-256-ofb no IV, different key │ 2FF57C092DC9 │
|
||||
│ aes-256-ofb with IV │ 5E6CB398F653 │
|
||||
│ aes-256-cbc no IV │ 1BC0629A92450D9E73A00E7D02CF4142 │
|
||||
└──────────────────────────────────┴──────────────────────────────────┘
|
||||
```
|
||||
|
||||
Пример в режиме `-gcm`:
|
||||
@ -116,9 +115,7 @@ SELECT comment, hex(secret) FROM encryption_test WHERE comment LIKE '%gcm%';
|
||||
|
||||
- aes-128-ecb, aes-192-ecb, aes-256-ecb
|
||||
- aes-128-cbc, aes-192-cbc, aes-256-cbc
|
||||
- aes-128-cfb1, aes-192-cfb1, aes-256-cfb1
|
||||
- aes-128-cfb8, aes-192-cfb8, aes-256-cfb8
|
||||
- aes-128-cfb128, aes-192-cfb128, aes-256-cfb128
|
||||
- aes-128-cfb128
|
||||
- aes-128-ofb, aes-192-ofb, aes-256-ofb
|
||||
|
||||
**Синтаксис**
|
||||
@ -145,7 +142,7 @@ aes_encrypt_mysql('mode', 'plaintext', 'key' [, iv])
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT encrypt('aes-256-cfb128', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv') = aes_encrypt_mysql('aes-256-cfb128', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv') AS ciphertexts_equal;
|
||||
SELECT encrypt('aes-256-ofb', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv') = aes_encrypt_mysql('aes-256-ofb', 'Secret', '12345678910121314151617181920212', 'iviviviviviviviv') AS ciphertexts_equal;
|
||||
```
|
||||
|
||||
Результат:
|
||||
@ -161,14 +158,14 @@ SELECT encrypt('aes-256-cfb128', 'Secret', '12345678910121314151617181920212', '
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT encrypt('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123');
|
||||
SELECT encrypt('aes-256-ofb', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123');
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
Received exception from server (version 21.1.2):
|
||||
Code: 36. DB::Exception: Received from localhost:9000. DB::Exception: Invalid key size: 33 expected 32: While processing encrypt('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123').
|
||||
Code: 36. DB::Exception: Received from localhost:9000. DB::Exception: Invalid key size: 33 expected 32: While processing encrypt('aes-256-ofb', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123').
|
||||
```
|
||||
|
||||
Однако функция `aes_encrypt_mysql` в аналогичном случае возвращает результат, который может быть обработан MySQL:
|
||||
@ -176,7 +173,7 @@ Code: 36. DB::Exception: Received from localhost:9000. DB::Exception: Invalid ke
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT hex(aes_encrypt_mysql('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123')) AS ciphertext;
|
||||
SELECT hex(aes_encrypt_mysql('aes-256-ofb', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123')) AS ciphertext;
|
||||
```
|
||||
|
||||
Результат:
|
||||
@ -192,7 +189,7 @@ SELECT hex(aes_encrypt_mysql('aes-256-cfb128', 'Secret', '1234567891012131415161
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT hex(aes_encrypt_mysql('aes-256-cfb128', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456')) AS ciphertext
|
||||
SELECT hex(aes_encrypt_mysql('aes-256-ofb', 'Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456')) AS ciphertext
|
||||
```
|
||||
|
||||
Результат:
|
||||
@ -206,7 +203,7 @@ SELECT hex(aes_encrypt_mysql('aes-256-cfb128', 'Secret', '1234567891012131415161
|
||||
Это совпадает с результатом, возвращаемым MySQL при таких же входящих значениях:
|
||||
|
||||
``` sql
|
||||
mysql> SET block_encryption_mode='aes-256-cfb128';
|
||||
mysql> SET block_encryption_mode='aes-256-ofb';
|
||||
Query OK, 0 rows affected (0.00 sec)
|
||||
|
||||
mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456') as ciphertext;
|
||||
@ -224,11 +221,10 @@ mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviv
|
||||
|
||||
- aes-128-ecb, aes-192-ecb, aes-256-ecb
|
||||
- aes-128-cbc, aes-192-cbc, aes-256-cbc
|
||||
- aes-128-cfb1, aes-192-cfb1, aes-256-cfb1
|
||||
- aes-128-cfb8, aes-192-cfb8, aes-256-cfb8
|
||||
- aes-128-cfb128, aes-192-cfb128, aes-256-cfb128
|
||||
- aes-128-cfb128
|
||||
- aes-128-ofb, aes-192-ofb, aes-256-ofb
|
||||
- aes-128-gcm, aes-192-gcm, aes-256-gcm
|
||||
- aes-128-ctr, aes-192-ctr, aes-256-ctr
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
@ -265,12 +261,12 @@ SELECT comment, hex(secret) FROM encryption_test;
|
||||
│ aes-256-gcm │ A8A3CCBC6426CFEEB60E4EAE03D3E94204C1B09E0254 │
|
||||
│ aes-256-gcm with AAD │ A8A3CCBC6426D9A1017A0A932322F1852260A4AD6837 │
|
||||
└──────────────────────┴──────────────────────────────────────────────┘
|
||||
┌─comment─────────────────────────────┬─hex(secret)──────────────────────┐
|
||||
│ aes-256-cfb128 no IV │ B4972BDC4459 │
|
||||
│ aes-256-cfb128 no IV, different key │ 2FF57C092DC9 │
|
||||
│ aes-256-cfb128 with IV │ 5E6CB398F653 │
|
||||
│ aes-256-cbc no IV │ 1BC0629A92450D9E73A00E7D02CF4142 │
|
||||
└─────────────────────────────────────┴──────────────────────────────────┘
|
||||
┌─comment──────────────────────────┬─hex(secret)──────────────────────┐
|
||||
│ aes-256-ofb no IV │ B4972BDC4459 │
|
||||
│ aes-256-ofb no IV, different key │ 2FF57C092DC9 │
|
||||
│ aes-256-ofb with IV │ 5E6CB398F653 │
|
||||
│ aes-256-cbc no IV │ 1BC0629A92450D9E73A00E7D02CF4142 │
|
||||
└──────────────────────────────────┴──────────────────────────────────┘
|
||||
```
|
||||
|
||||
Теперь попытаемся расшифровать эти данные:
|
||||
@ -278,19 +274,25 @@ SELECT comment, hex(secret) FROM encryption_test;
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT comment, decrypt('aes-256-cfb128', secret, '12345678910121314151617181920212') as plaintext FROM encryption_test;
|
||||
SELECT comment, decrypt('aes-256-ofb', secret, '12345678910121314151617181920212') as plaintext FROM encryption_test;
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
┌─comment─────────────────────────────┬─plaintext─┐
|
||||
│ aes-256-cfb128 no IV │ Secret │
|
||||
│ aes-256-cfb128 no IV, different key │ <20>4<EFBFBD>
|
||||
<20> │
|
||||
│ aes-256-cfb128 with IV │ <20><><EFBFBD>6<EFBFBD>~ │
|
||||
│aes-256-cbc no IV │ <20>2*4<>h3c<33>4w<34><77>@
|
||||
└─────────────────────────────────────┴───────────┘
|
||||
┌─comment──────────────┬─plaintext──┐
|
||||
│ aes-256-gcm │ OQ<4F>E
|
||||
<20>t<EFBFBD>7T<37>\<5C><><EFBFBD>\<5C> │
|
||||
│ aes-256-gcm with AAD │ OQ<4F>E
|
||||
<20>\<5C><>si<73><69><EFBFBD><EFBFBD>;<3B>o<EFBFBD><6F> │
|
||||
└──────────────────────┴────────────┘
|
||||
┌─comment──────────────────────────┬─plaintext─┐
|
||||
│ aes-256-ofb no IV │ Secret │
|
||||
│ aes-256-ofb no IV, different key │ <20>4<EFBFBD>
|
||||
<20> │
|
||||
│ aes-256-ofb with IV │ <20><><EFBFBD>6<EFBFBD>~ │
|
||||
│aes-256-cbc no IV │ <20>2*4<>h3c<33>4w<34><77>@
|
||||
└──────────────────────────────────┴───────────┘
|
||||
```
|
||||
|
||||
Обратите внимание, что только часть данных была расшифрована верно. Оставшаяся часть расшифрована некорректно, так как при шифровании использовались другие значения `mode`, `key`, или `iv`.
|
||||
@ -305,9 +307,7 @@ SELECT comment, decrypt('aes-256-cfb128', secret, '12345678910121314151617181920
|
||||
|
||||
- aes-128-ecb, aes-192-ecb, aes-256-ecb
|
||||
- aes-128-cbc, aes-192-cbc, aes-256-cbc
|
||||
- aes-128-cfb1, aes-192-cfb1, aes-256-cfb1
|
||||
- aes-128-cfb8, aes-192-cfb8, aes-256-cfb8
|
||||
- aes-128-cfb128, aes-192-cfb128, aes-256-cfb128
|
||||
- aes-128-cfb128
|
||||
- aes-128-ofb, aes-192-ofb, aes-256-ofb
|
||||
|
||||
**Синтаксис**
|
||||
@ -333,7 +333,7 @@ aes_decrypt_mysql('mode', 'ciphertext', 'key' [, iv])
|
||||
|
||||
|
||||
``` sql
|
||||
mysql> SET block_encryption_mode='aes-256-cfb128';
|
||||
mysql> SET block_encryption_mode='aes-256-ofb';
|
||||
Query OK, 0 rows affected (0.00 sec)
|
||||
|
||||
mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviviviviviviv123456') as ciphertext;
|
||||
@ -348,7 +348,7 @@ mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviv
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT aes_decrypt_mysql('aes-256-cfb128', unhex('24E9E4966469'), '123456789101213141516171819202122', 'iviviviviviviviv123456') AS plaintext;
|
||||
SELECT aes_decrypt_mysql('aes-256-ofb', unhex('24E9E4966469'), '123456789101213141516171819202122', 'iviviviviviviviv123456') AS plaintext;
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
@ -43,7 +43,7 @@ CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]na
|
||||
|
||||
При создании материализованного представления без использования `TO [db].[table]`, нужно обязательно указать `ENGINE` - движок таблицы для хранения данных.
|
||||
|
||||
При создании материализованного представления с испольованием `TO [db].[table]`, нельзя указывать `POPULATE`.
|
||||
При создании материализованного представления с использованием `TO [db].[table]`, нельзя указывать `POPULATE`.
|
||||
|
||||
Материализованное представление устроено следующим образом: при вставке данных в таблицу, указанную в SELECT-е, кусок вставляемых данных преобразуется этим запросом SELECT, и полученный результат вставляется в представление.
|
||||
|
||||
|
||||
@ -29,12 +29,12 @@ clickhouse-local --structure "table_structure" --input-format "format_of_incomin
|
||||
参数:
|
||||
|
||||
- `-S`, `--structure` — 输入数据的表结构。
|
||||
- `-if`, `--input-format` — 输入格式化类型, 默认是`TSV`。
|
||||
- `--input-format` — 输入格式化类型, 默认是`TSV`。
|
||||
- `-f`, `--file` — 数据路径, 默认是`stdin`。
|
||||
- `-q`, `--query` — 要查询的SQL语句使用`;`做分隔符。您必须指定`query`或`queries-file`选项。
|
||||
- `-qf`, `--queries-file` - 包含执行查询的文件路径。您必须指定`query`或`queries-file`选项。
|
||||
- `--queries-file` - 包含执行查询的文件路径。您必须指定`query`或`queries-file`选项。
|
||||
- `-N`, `--table` — 数据输出的表名，默认是`table`。
|
||||
- `-of`, `--format`, `--output-format` — 输出格式化类型, 默认是`TSV`。
|
||||
- `--format`, `--output-format` — 输出格式化类型, 默认是`TSV`。
|
||||
- `-d`, `--database` — 默认数据库名，默认是`_local`。
|
||||
- `--stacktrace` — 是否在出现异常时输出栈信息。
|
||||
- `--echo` — 执行前打印查询。
|
||||
@ -53,7 +53,7 @@ clickhouse-local --structure "table_structure" --input-format "format_of_incomin
|
||||
## 示例 {#examples}
|
||||
|
||||
``` bash
|
||||
echo -e "1,2\n3,4" | clickhouse-local -S "a Int64, b Int64" -if "CSV" -q "SELECT * FROM table"
|
||||
echo -e "1,2\n3,4" | clickhouse-local -S "a Int64, b Int64" --input-format "CSV" -q "SELECT * FROM table"
|
||||
Read 2 rows, 32.00 B in 0.000 sec., 5182 rows/sec., 80.97 KiB/sec.
|
||||
1 2
|
||||
3 4
|
||||
|
||||
@ -994,7 +994,7 @@ void Client::processConfig()
|
||||
/// The value of the option is used as the text of query (or of multiple queries).
|
||||
/// If stdin is not a terminal, INSERT data for the first query is read from it.
|
||||
/// - stdin is not a terminal. In this case queries are read from it.
|
||||
/// - -qf (--queries-file) command line option is present.
|
||||
/// - --queries-file command line option is present.
|
||||
/// The value of the option is used as file with query (or of multiple queries) to execute.
|
||||
|
||||
delayed_interactive = config().has("interactive") && (config().has("query") || config().has("queries-file"));
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <mutex>
|
||||
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <base/defines.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -24,9 +25,9 @@ public:
|
||||
|
||||
private:
|
||||
mutable std::mutex keeper_dispatcher_mutex;
|
||||
mutable std::shared_ptr<KeeperDispatcher> keeper_dispatcher;
|
||||
mutable std::shared_ptr<KeeperDispatcher> keeper_dispatcher TSA_GUARDED_BY(keeper_dispatcher_mutex);
|
||||
|
||||
ConfigurationPtr config;
|
||||
ConfigurationPtr config TSA_GUARDED_BY(keeper_dispatcher_mutex);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include "SharedLibraryHandler.h"
|
||||
#include <base/defines.h>
|
||||
|
||||
#include <unordered_map>
|
||||
#include <mutex>
|
||||
|
||||
@ -30,7 +32,7 @@ public:
|
||||
|
||||
private:
|
||||
/// map: dict_id -> sharedLibraryHandler
|
||||
std::unordered_map<std::string, SharedLibraryHandlerPtr> library_handlers;
|
||||
std::unordered_map<std::string, SharedLibraryHandlerPtr> library_handlers TSA_GUARDED_BY(mutex);
|
||||
std::mutex mutex;
|
||||
};
|
||||
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
#include <Client/ClientBase.h>
|
||||
#include <Client/LocalConnection.h>
|
||||
|
||||
#include <Common/ProgressIndication.h>
|
||||
#include <Common/StatusFile.h>
|
||||
#include <Common/InterruptListener.h>
|
||||
#include <Loggers/Loggers.h>
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <nanodbc/nanodbc.h>
|
||||
#include <mutex>
|
||||
#include <base/BorrowedObjectPool.h>
|
||||
#include <base/defines.h>
|
||||
#include <unordered_map>
|
||||
|
||||
|
||||
@ -165,7 +166,7 @@ public:
|
||||
private:
|
||||
/// [connection_settings_string] -> [connection_pool]
|
||||
using PoolFactory = std::unordered_map<std::string, nanodbc::PoolPtr>;
|
||||
PoolFactory factory;
|
||||
PoolFactory factory TSA_GUARDED_BY(mutex);
|
||||
std::mutex mutex;
|
||||
};
|
||||
|
||||
|
||||
@ -1515,7 +1515,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
|
||||
/// Init trace collector only after trace_log system table was created
|
||||
/// Disable it if we collect test coverage information, because it will work extremely slow.
|
||||
#if USE_UNWIND && !WITH_COVERAGE && defined(__x86_64__)
|
||||
#if USE_UNWIND && !WITH_COVERAGE
|
||||
/// Profilers cannot work reliably with any other libunwind or without PHDR cache.
|
||||
if (hasPHDRCache())
|
||||
{
|
||||
|
||||
@ -386,6 +386,39 @@
|
||||
text-align: center;
|
||||
margin-top: 5em;
|
||||
}
|
||||
|
||||
#chart
|
||||
{
|
||||
background-color: var(--element-background-color);
|
||||
filter: drop-shadow(.2rem .2rem .2rem var(--shadow-color));
|
||||
display: none;
|
||||
height: 70vh;
|
||||
}
|
||||
|
||||
/* This is for charts (uPlot), Copyright (c) 2022 Leon Sorokin, MIT License, https://github.com/leeoniya/uPlot/ */
|
||||
.u-wrap {position: relative;user-select: none;}
|
||||
.u-over, .u-under, .u-axis {position: absolute;}
|
||||
.u-under {overflow: hidden;}
|
||||
.uplot canvas {display: block;position: relative;width: 100%;height: 100%;}
|
||||
.u-legend {margin: auto;text-align: center; margin-top: 1em; font-family: Liberation Mono, DejaVu Sans Mono, MonoLisa, Consolas, monospace;}
|
||||
.u-inline {display: block;}
|
||||
.u-inline * {display: inline-block;}
|
||||
.u-inline tr {margin-right: 16px;}
|
||||
.u-legend th {font-weight: 600;}
|
||||
.u-legend th > * {vertical-align: middle;display: inline-block;}
|
||||
.u-legend td { min-width: 13em; }
|
||||
.u-legend .u-marker {width: 1em;height: 1em;margin-right: 4px;background-clip: padding-box !important;}
|
||||
.u-inline.u-live th::after {content: ":";vertical-align: middle;}
|
||||
.u-inline:not(.u-live) .u-value {display: none;}
|
||||
.u-series > * {padding: 4px;}
|
||||
.u-series th {cursor: pointer;}
|
||||
.u-legend .u-off > * {opacity: 0.3;}
|
||||
.u-select {background: rgba(0,0,0,0.07);position: absolute;pointer-events: none;}
|
||||
.u-cursor-x, .u-cursor-y {position: absolute;left: 0;top: 0;pointer-events: none;will-change: transform;z-index: 100;}
|
||||
.u-hz .u-cursor-x, .u-vt .u-cursor-y {height: 100%;border-right: 1px dashed #607D8B;}
|
||||
.u-hz .u-cursor-y, .u-vt .u-cursor-x {width: 100%;border-bottom: 1px dashed #607D8B;}
|
||||
.u-cursor-pt {position: absolute;top: 0;left: 0;border-radius: 50%;border: 0 solid;pointer-events: none;will-change: transform;z-index: 100;/*this has to be !important since we set inline "background" shorthand */background-clip: padding-box !important;}
|
||||
.u-axis.u-off, .u-select.u-off, .u-cursor-x.u-off, .u-cursor-y.u-off, .u-cursor-pt.u-off {display: none;}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
@ -410,6 +443,7 @@
|
||||
<table class="monospace-table shadow" id="data-table"></table>
|
||||
<pre class="monospace-table shadow" id="data-unparsed"></pre>
|
||||
</div>
|
||||
<div id="chart"></div>
|
||||
<svg id="graph" fill="none"></svg>
|
||||
<p id="error" class="monospace shadow">
|
||||
</p>
|
||||
@ -530,8 +564,13 @@
|
||||
if (status === 200) {
|
||||
let json;
|
||||
try { json = JSON.parse(response); } catch (e) {}
|
||||
|
||||
if (json !== undefined && json.statistics !== undefined) {
|
||||
renderResult(json);
|
||||
} else if (Array.isArray(json) && json.length == 2 &&
|
||||
Array.isArray(json[0]) && Array.isArray(json[1]) && json[0].length > 1 && json[0].length == json[1].length) {
|
||||
/// If user requested FORMAT JSONCompactColumns, we will render it as a chart.
|
||||
renderChart(json);
|
||||
} else {
|
||||
renderUnparsedResult(response);
|
||||
}
|
||||
@ -578,30 +617,27 @@
|
||||
}
|
||||
}
|
||||
|
||||
function clearElement(id)
|
||||
{
|
||||
let elem = document.getElementById(id);
|
||||
while (elem.firstChild) {
|
||||
elem.removeChild(elem.lastChild);
|
||||
}
|
||||
elem.style.display = 'none';
|
||||
}
|
||||
|
||||
function clear()
|
||||
{
|
||||
let table = document.getElementById('data-table');
|
||||
while (table.firstChild) {
|
||||
table.removeChild(table.lastChild);
|
||||
}
|
||||
|
||||
let graph = document.getElementById('graph');
|
||||
while (graph.firstChild) {
|
||||
graph.removeChild(graph.lastChild);
|
||||
}
|
||||
graph.style.display = 'none';
|
||||
|
||||
document.getElementById('data-unparsed').innerText = '';
|
||||
document.getElementById('data-unparsed').style.display = 'none';
|
||||
|
||||
document.getElementById('error').innerText = '';
|
||||
document.getElementById('error').style.display = 'none';
|
||||
clearElement('data-table');
|
||||
clearElement('graph');
|
||||
clearElement('chart');
|
||||
clearElement('data-unparsed');
|
||||
clearElement('error');
|
||||
clearElement('hourglass');
|
||||
|
||||
document.getElementById('check-mark').innerText = '';
|
||||
document.getElementById('hourglass').innerText = '';
|
||||
document.getElementById('stats').innerText = '';
|
||||
|
||||
document.getElementById('hourglass').style.display = 'none';
|
||||
document.getElementById('check-mark').style.display = 'none';
|
||||
|
||||
document.getElementById('logo-container').style.display = 'block';
|
||||
}
|
||||
|
||||
@ -738,6 +774,7 @@
|
||||
}
|
||||
let table = document.getElementById('data-table');
|
||||
table.appendChild(tbody);
|
||||
table.style.display = 'table';
|
||||
}
|
||||
|
||||
function renderTable(response)
|
||||
@ -792,6 +829,7 @@
|
||||
let table = document.getElementById('data-table');
|
||||
table.appendChild(thead);
|
||||
table.appendChild(tbody);
|
||||
table.style.display = 'table';
|
||||
}
|
||||
|
||||
/// A function to render raw data when non-default format is specified.
|
||||
@ -873,16 +911,80 @@
|
||||
svg.style.height = graph.graph().height;
|
||||
}
|
||||
|
||||
function setColorTheme(theme) {
|
||||
window.localStorage.setItem('theme', theme);
|
||||
document.documentElement.setAttribute('data-theme', theme);
|
||||
let load_uplot_promise;
|
||||
function loadUplot() {
|
||||
if (load_uplot_promise) { return load_uplot_promise; }
|
||||
load_uplot_promise = loadJS('https://cdn.jsdelivr.net/npm/uplot@1.6.21/dist/uPlot.iife.min.js',
|
||||
'sha384-TwdJPnTsKP6pnvFZZKda0WJCXpjcHCa7MYHmjrYDu6rsEsb/UnFdoL0phS5ODqTA');
|
||||
return load_uplot_promise;
|
||||
}
|
||||
|
||||
let uplot;
|
||||
async function renderChart(json)
|
||||
{
|
||||
await loadUplot();
|
||||
clear();
|
||||
|
||||
let chart = document.getElementById('chart');
|
||||
chart.style.display = 'block';
|
||||
|
||||
let paths = uPlot.paths.stepped({align: 1});
|
||||
|
||||
const [line_color, fill_color, grid_color, axes_color] = theme == 'light'
|
||||
? ["#F80", "#FED", "#c7d0d9", "#2c3235"]
|
||||
: ["#888", "#045", "#2c3235", "#c7d0d9"];
|
||||
|
||||
const opts = {
|
||||
width: chart.clientWidth,
|
||||
height: chart.clientHeight,
|
||||
scales: { x: { time: json[0][0] > 1000000000 && json[0][0] < 2000000000 } },
|
||||
axes: [ { stroke: axes_color,
|
||||
grid: { width: 1 / devicePixelRatio, stroke: grid_color },
|
||||
ticks: { width: 1 / devicePixelRatio, stroke: grid_color } },
|
||||
{ stroke: axes_color,
|
||||
grid: { width: 1 / devicePixelRatio, stroke: grid_color },
|
||||
ticks: { width: 1 / devicePixelRatio, stroke: grid_color } } ],
|
||||
series: [ { label: "x" },
|
||||
{ label: "y", stroke: line_color, fill: fill_color,
|
||||
drawStyle: 0, lineInterpolation: 1, paths } ],
|
||||
padding: [ null, null, null, (Math.ceil(Math.log10(Math.max(...json[1]))) + Math.floor(Math.log10(Math.max(...json[1])) / 3)) * 6 ],
|
||||
};
|
||||
|
||||
uplot = new uPlot(opts, json, chart);
|
||||
}
|
||||
|
||||
function resizeChart() {
|
||||
if (uplot) {
|
||||
let chart = document.getElementById('chart');
|
||||
uplot.setSize({ width: chart.clientWidth, height: chart.clientHeight });
|
||||
}
|
||||
}
|
||||
|
||||
function redrawChart() {
|
||||
if (uplot && document.getElementById('chart').style.display == 'block') {
|
||||
renderChart(uplot.data);
|
||||
}
|
||||
}
|
||||
|
||||
new ResizeObserver(resizeChart).observe(document.getElementById('chart'));
|
||||
|
||||
/// First we check if theme is set via the 'theme' GET parameter, if not, we check localStorage, otherwise we check OS preference.
|
||||
let theme = current_url.searchParams.get('theme');
|
||||
if (['dark', 'light'].indexOf(theme) === -1) {
|
||||
theme = window.localStorage.getItem('theme');
|
||||
}
|
||||
if (!theme) {
|
||||
theme = 'light';
|
||||
}
|
||||
|
||||
function setColorTheme(new_theme, update_preference) {
|
||||
theme = new_theme;
|
||||
if (update_preference) {
|
||||
window.localStorage.setItem('theme', theme);
|
||||
}
|
||||
document.documentElement.setAttribute('data-theme', theme);
|
||||
redrawChart();
|
||||
}
|
||||
|
||||
if (theme) {
|
||||
document.documentElement.setAttribute('data-theme', theme);
|
||||
@ -890,26 +992,21 @@
|
||||
/// Obtain system-level user preference
|
||||
const media_query_list = window.matchMedia('(prefers-color-scheme: dark)');
|
||||
if (media_query_list.matches) {
|
||||
/// Set without saving to localstorage
|
||||
document.documentElement.setAttribute('data-theme', 'dark');
|
||||
setColorTheme('dark');
|
||||
}
|
||||
|
||||
/// There is a rumor that on some computers, the theme is changing automatically on day/night.
|
||||
media_query_list.addEventListener('change', function(e) {
|
||||
if (e.matches) {
|
||||
document.documentElement.setAttribute('data-theme', 'dark');
|
||||
} else {
|
||||
document.documentElement.setAttribute('data-theme', 'light');
|
||||
}
|
||||
setColorTheme(e.matches ? 'dark' : 'light');
|
||||
});
|
||||
}
|
||||
|
||||
document.getElementById('toggle-light').onclick = function() {
|
||||
setColorTheme('light');
|
||||
setColorTheme('light', true);
|
||||
}
|
||||
|
||||
document.getElementById('toggle-dark').onclick = function() {
|
||||
setColorTheme('dark');
|
||||
setColorTheme('dark', true);
|
||||
}
|
||||
</script>
|
||||
</html>
|
||||
|
||||
358
src/Access/AccessBackup.cpp
Normal file
358
src/Access/AccessBackup.cpp
Normal file
@ -0,0 +1,358 @@
|
||||
#include <Access/AccessBackup.h>
|
||||
#include <Access/AccessControl.h>
|
||||
#include <Access/AccessEntityIO.h>
|
||||
#include <Access/Common/AccessRightsElement.h>
|
||||
#include <Access/User.h>
|
||||
#include <Access/Role.h>
|
||||
#include <Access/SettingsProfile.h>
|
||||
#include <Access/RowPolicy.h>
|
||||
#include <Access/Quota.h>
|
||||
#include <Backups/BackupEntriesCollector.h>
|
||||
#include <Backups/BackupEntryFromMemory.h>
|
||||
#include <Backups/IBackup.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <Poco/UUIDGenerator.h>
|
||||
#include <base/insertAtEnd.h>
|
||||
#include <boost/range/algorithm/copy.hpp>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
/// Represents a list of access entities as they're stored in a backup.
|
||||
struct AccessEntitiesInBackup
|
||||
{
|
||||
std::unordered_map<UUID, AccessEntityPtr> entities;
|
||||
std::unordered_map<UUID, std::pair<String, AccessEntityType>> dependencies;
|
||||
|
||||
BackupEntryPtr toBackupEntry() const
|
||||
{
|
||||
WriteBufferFromOwnString buf;
|
||||
|
||||
for (const auto & [id, entity] : entities)
|
||||
{
|
||||
writeText(id, buf);
|
||||
writeChar('\t', buf);
|
||||
writeText(entity->getTypeInfo().name, buf);
|
||||
writeChar('\t', buf);
|
||||
writeText(entity->getName(), buf);
|
||||
writeChar('\n', buf);
|
||||
writeText(serializeAccessEntity(*entity), buf);
|
||||
writeChar('\n', buf);
|
||||
}
|
||||
|
||||
if (!dependencies.empty())
|
||||
{
|
||||
writeText("DEPENDENCIES\n", buf);
|
||||
for (const auto & [id, name_and_type] : dependencies)
|
||||
{
|
||||
writeText(id, buf);
|
||||
writeChar('\t', buf);
|
||||
writeText(AccessEntityTypeInfo::get(name_and_type.second).name, buf);
|
||||
writeChar('\t', buf);
|
||||
writeText(name_and_type.first, buf);
|
||||
writeChar('\n', buf);
|
||||
}
|
||||
}
|
||||
|
||||
return std::make_shared<BackupEntryFromMemory>(buf.str());
|
||||
}
|
||||
|
||||
static AccessEntitiesInBackup fromBackupEntry(const IBackupEntry & backup_entry, const String & file_path)
|
||||
{
|
||||
try
|
||||
{
|
||||
AccessEntitiesInBackup res;
|
||||
std::unique_ptr<ReadBuffer> buf = backup_entry.getReadBuffer();
|
||||
|
||||
bool dependencies_found = false;
|
||||
|
||||
while (!buf->eof())
|
||||
{
|
||||
String line;
|
||||
readStringUntilNewlineInto(line, *buf);
|
||||
buf->ignore();
|
||||
if (line == "DEPENDENCIES")
|
||||
{
|
||||
dependencies_found = true;
|
||||
break;
|
||||
}
|
||||
|
||||
size_t id_endpos = line.find('\t');
|
||||
String id_as_string = line.substr(0, id_endpos);
|
||||
UUID id = parse<UUID>(line);
|
||||
line.clear();
|
||||
|
||||
String queries;
|
||||
while (!buf->eof())
|
||||
{
|
||||
String query;
|
||||
readStringUntilNewlineInto(query, *buf);
|
||||
buf->ignore();
|
||||
if (query.empty())
|
||||
break;
|
||||
if (!queries.empty())
|
||||
queries.append("\n");
|
||||
queries.append(query);
|
||||
}
|
||||
|
||||
AccessEntityPtr entity = deserializeAccessEntity(queries);
|
||||
res.entities.emplace(id, entity);
|
||||
}
|
||||
|
||||
if (dependencies_found)
|
||||
{
|
||||
while (!buf->eof())
|
||||
{
|
||||
String id_as_string;
|
||||
readStringInto(id_as_string, *buf);
|
||||
buf->ignore();
|
||||
UUID id = parse<UUID>(id_as_string);
|
||||
|
||||
String type_as_string;
|
||||
readStringInto(type_as_string, *buf);
|
||||
buf->ignore();
|
||||
AccessEntityType type = AccessEntityTypeInfo::parseType(type_as_string);
|
||||
|
||||
String name;
|
||||
readStringInto(name, *buf);
|
||||
buf->ignore();
|
||||
|
||||
if (!res.entities.contains(id))
|
||||
res.dependencies.emplace(id, std::pair{name, type});
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
e.addMessage("While parsing " + file_path);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<UUID> findDependencies(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities)
|
||||
{
|
||||
std::vector<UUID> res;
|
||||
for (const auto & entity : entities | boost::adaptors::map_values)
|
||||
insertAtEnd(res, entity->findDependencies());
|
||||
|
||||
/// Remove duplicates in the list of dependencies (some entities can refer to other entities).
|
||||
::sort(res.begin(), res.end());
|
||||
res.erase(std::unique(res.begin(), res.end()), res.end());
|
||||
for (const auto & id : entities | boost::adaptors::map_keys)
|
||||
{
|
||||
auto it = std::lower_bound(res.begin(), res.end(), id);
|
||||
if ((it != res.end()) && (*it == id))
|
||||
res.erase(it);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
std::unordered_map<UUID, std::pair<String, AccessEntityType>> readDependenciesNamesAndTypes(const std::vector<UUID> & dependencies, const AccessControl & access_control)
|
||||
{
|
||||
std::unordered_map<UUID, std::pair<String, AccessEntityType>> res;
|
||||
for (const auto & id : dependencies)
|
||||
{
|
||||
if (auto name_and_type = access_control.tryReadNameWithType(id))
|
||||
res.emplace(id, name_and_type.value());
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
std::unordered_map<UUID, UUID> resolveDependencies(const std::unordered_map<UUID, std::pair<String, AccessEntityType>> & dependencies, const AccessControl & access_control, bool allow_unresolved_dependencies)
|
||||
{
|
||||
std::unordered_map<UUID, UUID> old_to_new_ids;
|
||||
for (const auto & [id, name_and_type] : dependencies)
|
||||
{
|
||||
std::optional<UUID> new_id;
|
||||
if (allow_unresolved_dependencies)
|
||||
new_id = access_control.find(name_and_type.second, name_and_type.first);
|
||||
else
|
||||
new_id = access_control.getID(name_and_type.second, name_and_type.first);
|
||||
if (new_id)
|
||||
old_to_new_ids.emplace(id, *new_id);
|
||||
}
|
||||
return old_to_new_ids;
|
||||
}
|
||||
|
||||
void generateRandomIDs(std::vector<std::pair<UUID, AccessEntityPtr>> & entities, std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||
{
|
||||
Poco::UUIDGenerator generator;
|
||||
for (auto & [id, entity] : entities)
|
||||
{
|
||||
UUID new_id;
|
||||
generator.createRandom().copyTo(reinterpret_cast<char *>(&new_id));
|
||||
old_to_new_ids.emplace(id, new_id);
|
||||
id = new_id;
|
||||
}
|
||||
}
|
||||
|
||||
void replaceDependencies(std::vector<std::pair<UUID, AccessEntityPtr>> & entities, const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||
{
|
||||
for (auto & entity : entities | boost::adaptors::map_values)
|
||||
{
|
||||
bool need_replace = false;
|
||||
for (const auto & dependency : entity->findDependencies())
|
||||
{
|
||||
if (old_to_new_ids.contains(dependency))
|
||||
{
|
||||
need_replace = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!need_replace)
|
||||
continue;
|
||||
|
||||
auto new_entity = entity->clone();
|
||||
new_entity->replaceDependencies(old_to_new_ids);
|
||||
entity = new_entity;
|
||||
}
|
||||
}
|
||||
|
||||
AccessRightsElements getRequiredAccessToRestore(const std::unordered_map<UUID, AccessEntityPtr> & entities)
|
||||
{
|
||||
AccessRightsElements res;
|
||||
for (const auto & entity : entities | boost::adaptors::map_values)
|
||||
{
|
||||
auto entity_type = entity->getType();
|
||||
switch (entity_type)
|
||||
{
|
||||
case User::TYPE:
|
||||
{
|
||||
const auto & user = typeid_cast<const User &>(*entity);
|
||||
res.emplace_back(AccessType::CREATE_USER);
|
||||
auto elements = user.access.getElements();
|
||||
for (auto & element : elements)
|
||||
{
|
||||
if (element.is_partial_revoke)
|
||||
continue;
|
||||
element.grant_option = true;
|
||||
res.emplace_back(element);
|
||||
}
|
||||
if (!user.granted_roles.isEmpty())
|
||||
res.emplace_back(AccessType::ROLE_ADMIN);
|
||||
break;
|
||||
}
|
||||
|
||||
case Role::TYPE:
|
||||
{
|
||||
const auto & role = typeid_cast<const Role &>(*entity);
|
||||
res.emplace_back(AccessType::CREATE_ROLE);
|
||||
auto elements = role.access.getElements();
|
||||
for (auto & element : elements)
|
||||
{
|
||||
if (element.is_partial_revoke)
|
||||
continue;
|
||||
element.grant_option = true;
|
||||
res.emplace_back(element);
|
||||
}
|
||||
if (!role.granted_roles.isEmpty())
|
||||
res.emplace_back(AccessType::ROLE_ADMIN);
|
||||
break;
|
||||
}
|
||||
|
||||
case SettingsProfile::TYPE:
|
||||
{
|
||||
res.emplace_back(AccessType::CREATE_SETTINGS_PROFILE);
|
||||
break;
|
||||
}
|
||||
|
||||
case RowPolicy::TYPE:
|
||||
{
|
||||
const auto & policy = typeid_cast<const RowPolicy &>(*entity);
|
||||
res.emplace_back(AccessType::CREATE_ROW_POLICY, policy.getDatabase(), policy.getTableName());
|
||||
break;
|
||||
}
|
||||
|
||||
case Quota::TYPE:
|
||||
{
|
||||
res.emplace_back(AccessType::CREATE_QUOTA);
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
throw Exception("Unknown type: " + toString(entity_type), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
void backupAccessEntities(
|
||||
BackupEntriesCollector & backup_entries_collector,
|
||||
const String & data_path_in_backup,
|
||||
const AccessControl & access_control,
|
||||
AccessEntityType type)
|
||||
{
|
||||
auto entities = access_control.readAllForBackup(type, backup_entries_collector.getBackupSettings());
|
||||
auto dependencies = readDependenciesNamesAndTypes(findDependencies(entities), access_control);
|
||||
AccessEntitiesInBackup ab;
|
||||
boost::range::copy(entities, std::inserter(ab.entities, ab.entities.end()));
|
||||
ab.dependencies = std::move(dependencies);
|
||||
backup_entries_collector.addBackupEntry(fs::path{data_path_in_backup} / "access.txt", ab.toBackupEntry());
|
||||
}
|
||||
|
||||
|
||||
AccessRestoreTask::AccessRestoreTask(
|
||||
const BackupPtr & backup_, const RestoreSettings & restore_settings_, std::shared_ptr<IRestoreCoordination> restore_coordination_)
|
||||
: backup(backup_), restore_settings(restore_settings_), restore_coordination(restore_coordination_)
|
||||
{
|
||||
}
|
||||
|
||||
AccessRestoreTask::~AccessRestoreTask() = default;
|
||||
|
||||
void AccessRestoreTask::addDataPath(const String & data_path)
|
||||
{
|
||||
if (!data_paths.emplace(data_path).second)
|
||||
return;
|
||||
|
||||
String file_path = fs::path{data_path} / "access.txt";
|
||||
auto backup_entry = backup->readFile(file_path);
|
||||
auto ab = AccessEntitiesInBackup::fromBackupEntry(*backup_entry, file_path);
|
||||
|
||||
boost::range::copy(ab.entities, std::inserter(entities, entities.end()));
|
||||
boost::range::copy(ab.dependencies, std::inserter(dependencies, dependencies.end()));
|
||||
for (const auto & id : entities | boost::adaptors::map_keys)
|
||||
dependencies.erase(id);
|
||||
}
|
||||
|
||||
bool AccessRestoreTask::hasDataPath(const String & data_path) const
|
||||
{
|
||||
return data_paths.contains(data_path);
|
||||
}
|
||||
|
||||
AccessRightsElements AccessRestoreTask::getRequiredAccess() const
|
||||
{
|
||||
return getRequiredAccessToRestore(entities);
|
||||
}
|
||||
|
||||
void AccessRestoreTask::restore(AccessControl & access_control) const
|
||||
{
|
||||
auto old_to_new_ids = resolveDependencies(dependencies, access_control, restore_settings.allow_unresolved_access_dependencies);
|
||||
|
||||
std::vector<std::pair<UUID, AccessEntityPtr>> new_entities;
|
||||
boost::range::copy(entities, std::back_inserter(new_entities));
|
||||
generateRandomIDs(new_entities, old_to_new_ids);
|
||||
|
||||
replaceDependencies(new_entities, old_to_new_ids);
|
||||
|
||||
access_control.insertFromBackup(new_entities, restore_settings, restore_coordination);
|
||||
}
|
||||
|
||||
}
|
||||
56
src/Access/AccessBackup.h
Normal file
56
src/Access/AccessBackup.h
Normal file
@ -0,0 +1,56 @@
|
||||
#pragma once
|
||||
|
||||
#include <Backups/RestoreSettings.h>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
class AccessControl;
|
||||
enum class AccessEntityType;
|
||||
class BackupEntriesCollector;
|
||||
class RestorerFromBackup;
|
||||
class IBackup;
|
||||
using BackupPtr = std::shared_ptr<const IBackup>;
|
||||
class IRestoreCoordination;
|
||||
struct IAccessEntity;
|
||||
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
|
||||
class AccessRightsElements;
|
||||
|
||||
|
||||
/// Makes a backup of access entities of a specified type.
|
||||
void backupAccessEntities(
|
||||
BackupEntriesCollector & backup_entries_collector,
|
||||
const String & data_path_in_backup,
|
||||
const AccessControl & access_control,
|
||||
AccessEntityType type);
|
||||
|
||||
/// Restores access entities from a backup.
|
||||
class AccessRestoreTask
|
||||
{
|
||||
public:
|
||||
AccessRestoreTask(
|
||||
const BackupPtr & backup_, const RestoreSettings & restore_settings_, std::shared_ptr<IRestoreCoordination> restore_coordination_);
|
||||
~AccessRestoreTask();
|
||||
|
||||
/// Adds a data path to loads access entities from.
|
||||
void addDataPath(const String & data_path);
|
||||
bool hasDataPath(const String & data_path) const;
|
||||
|
||||
/// Checks that the current user can do restoring.
|
||||
AccessRightsElements getRequiredAccess() const;
|
||||
|
||||
/// Inserts all access entities loaded from all the paths added by addDataPath().
|
||||
void restore(AccessControl & access_control) const;
|
||||
|
||||
private:
|
||||
BackupPtr backup;
|
||||
RestoreSettings restore_settings;
|
||||
std::shared_ptr<IRestoreCoordination> restore_coordination;
|
||||
std::unordered_map<UUID, AccessEntityPtr> entities;
|
||||
std::unordered_map<UUID, std::pair<String, AccessEntityType>> dependencies;
|
||||
std::unordered_set<String> data_paths;
|
||||
};
|
||||
|
||||
}
|
||||
@ -15,7 +15,11 @@
|
||||
#include <Access/User.h>
|
||||
#include <Access/ExternalAuthenticators.h>
|
||||
#include <Access/AccessChangesNotifier.h>
|
||||
#include <Access/AccessBackup.h>
|
||||
#include <Backups/BackupEntriesCollector.h>
|
||||
#include <Backups/RestorerFromBackup.h>
|
||||
#include <Core/Settings.h>
|
||||
#include <base/defines.h>
|
||||
#include <base/find_symbols.h>
|
||||
#include <Poco/AccessExpireCache.h>
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
@ -130,7 +134,7 @@ public:
|
||||
}
|
||||
|
||||
private:
|
||||
Strings registered_prefixes;
|
||||
Strings registered_prefixes TSA_GUARDED_BY(mutex);
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
|
||||
@ -184,39 +188,25 @@ void AccessControl::setUsersConfig(const Poco::Util::AbstractConfiguration & use
|
||||
return;
|
||||
}
|
||||
}
|
||||
addUsersConfigStorage(users_config_);
|
||||
addUsersConfigStorage(UsersConfigAccessStorage::STORAGE_TYPE, users_config_, false);
|
||||
}
|
||||
|
||||
void AccessControl::addUsersConfigStorage(const Poco::Util::AbstractConfiguration & users_config_)
|
||||
void AccessControl::addUsersConfigStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & users_config_, bool allow_backup_)
|
||||
{
|
||||
addUsersConfigStorage(UsersConfigAccessStorage::STORAGE_TYPE, users_config_);
|
||||
}
|
||||
|
||||
void AccessControl::addUsersConfigStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & users_config_)
|
||||
{
|
||||
auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, *this);
|
||||
auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, *this, allow_backup_);
|
||||
new_storage->setConfig(users_config_);
|
||||
addStorage(new_storage);
|
||||
LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}",
|
||||
String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath());
|
||||
}
|
||||
|
||||
void AccessControl::addUsersConfigStorage(
|
||||
const String & users_config_path_,
|
||||
const String & include_from_path_,
|
||||
const String & preprocessed_dir_,
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_)
|
||||
{
|
||||
addUsersConfigStorage(
|
||||
UsersConfigAccessStorage::STORAGE_TYPE, users_config_path_, include_from_path_, preprocessed_dir_, get_zookeeper_function_);
|
||||
}
|
||||
|
||||
void AccessControl::addUsersConfigStorage(
|
||||
const String & storage_name_,
|
||||
const String & users_config_path_,
|
||||
const String & include_from_path_,
|
||||
const String & preprocessed_dir_,
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_)
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_,
|
||||
bool allow_backup_)
|
||||
{
|
||||
auto storages = getStoragesPtr();
|
||||
for (const auto & storage : *storages)
|
||||
@ -227,7 +217,7 @@ void AccessControl::addUsersConfigStorage(
|
||||
return;
|
||||
}
|
||||
}
|
||||
auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, *this);
|
||||
auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, *this, allow_backup_);
|
||||
new_storage->load(users_config_path_, include_from_path_, preprocessed_dir_, get_zookeeper_function_);
|
||||
addStorage(new_storage);
|
||||
LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath());
|
||||
@ -237,7 +227,8 @@ void AccessControl::addUsersConfigStorage(
|
||||
void AccessControl::addReplicatedStorage(
|
||||
const String & storage_name_,
|
||||
const String & zookeeper_path_,
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_)
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_,
|
||||
bool allow_backup_)
|
||||
{
|
||||
auto storages = getStoragesPtr();
|
||||
for (const auto & storage : *storages)
|
||||
@ -245,17 +236,12 @@ void AccessControl::addReplicatedStorage(
|
||||
if (auto replicated_storage = typeid_cast<std::shared_ptr<ReplicatedAccessStorage>>(storage))
|
||||
return;
|
||||
}
|
||||
auto new_storage = std::make_shared<ReplicatedAccessStorage>(storage_name_, zookeeper_path_, get_zookeeper_function_, *changes_notifier);
|
||||
auto new_storage = std::make_shared<ReplicatedAccessStorage>(storage_name_, zookeeper_path_, get_zookeeper_function_, *changes_notifier, allow_backup_);
|
||||
addStorage(new_storage);
|
||||
LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName());
|
||||
}
|
||||
|
||||
void AccessControl::addDiskStorage(const String & directory_, bool readonly_)
|
||||
{
|
||||
addDiskStorage(DiskAccessStorage::STORAGE_TYPE, directory_, readonly_);
|
||||
}
|
||||
|
||||
void AccessControl::addDiskStorage(const String & storage_name_, const String & directory_, bool readonly_)
|
||||
void AccessControl::addDiskStorage(const String & storage_name_, const String & directory_, bool readonly_, bool allow_backup_)
|
||||
{
|
||||
auto storages = getStoragesPtr();
|
||||
for (const auto & storage : *storages)
|
||||
@ -270,13 +256,13 @@ void AccessControl::addDiskStorage(const String & storage_name_, const String &
|
||||
}
|
||||
}
|
||||
}
|
||||
auto new_storage = std::make_shared<DiskAccessStorage>(storage_name_, directory_, readonly_, *changes_notifier);
|
||||
auto new_storage = std::make_shared<DiskAccessStorage>(storage_name_, directory_, *changes_notifier, readonly_, allow_backup_);
|
||||
addStorage(new_storage);
|
||||
LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath());
|
||||
}
|
||||
|
||||
|
||||
void AccessControl::addMemoryStorage(const String & storage_name_)
|
||||
void AccessControl::addMemoryStorage(const String & storage_name_, bool allow_backup_)
|
||||
{
|
||||
auto storages = getStoragesPtr();
|
||||
for (const auto & storage : *storages)
|
||||
@ -284,7 +270,7 @@ void AccessControl::addMemoryStorage(const String & storage_name_)
|
||||
if (auto memory_storage = typeid_cast<std::shared_ptr<MemoryAccessStorage>>(storage))
|
||||
return;
|
||||
}
|
||||
auto new_storage = std::make_shared<MemoryAccessStorage>(storage_name_, *changes_notifier);
|
||||
auto new_storage = std::make_shared<MemoryAccessStorage>(storage_name_, *changes_notifier, allow_backup_);
|
||||
addStorage(new_storage);
|
||||
LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName());
|
||||
}
|
||||
@ -327,20 +313,23 @@ void AccessControl::addStoragesFromUserDirectoriesConfig(
|
||||
|
||||
if (type == MemoryAccessStorage::STORAGE_TYPE)
|
||||
{
|
||||
addMemoryStorage(name);
|
||||
bool allow_backup = config.getBool(prefix + ".allow_backup", true);
|
||||
addMemoryStorage(name, allow_backup);
|
||||
}
|
||||
else if (type == UsersConfigAccessStorage::STORAGE_TYPE)
|
||||
{
|
||||
String path = config.getString(prefix + ".path");
|
||||
if (std::filesystem::path{path}.is_relative() && std::filesystem::exists(config_dir + path))
|
||||
path = config_dir + path;
|
||||
addUsersConfigStorage(name, path, include_from_path, dbms_dir, get_zookeeper_function);
|
||||
bool allow_backup = config.getBool(prefix + ".allow_backup", false); /// We don't backup users.xml by default.
|
||||
addUsersConfigStorage(name, path, include_from_path, dbms_dir, get_zookeeper_function, allow_backup);
|
||||
}
|
||||
else if (type == DiskAccessStorage::STORAGE_TYPE)
|
||||
{
|
||||
String path = config.getString(prefix + ".path");
|
||||
bool readonly = config.getBool(prefix + ".readonly", false);
|
||||
addDiskStorage(name, path, readonly);
|
||||
bool allow_backup = config.getBool(prefix + ".allow_backup", true);
|
||||
addDiskStorage(name, path, readonly, allow_backup);
|
||||
}
|
||||
else if (type == LDAPAccessStorage::STORAGE_TYPE)
|
||||
{
|
||||
@ -349,7 +338,8 @@ void AccessControl::addStoragesFromUserDirectoriesConfig(
|
||||
else if (type == ReplicatedAccessStorage::STORAGE_TYPE)
|
||||
{
|
||||
String zookeeper_path = config.getString(prefix + ".zookeeper_path");
|
||||
addReplicatedStorage(name, zookeeper_path, get_zookeeper_function);
|
||||
bool allow_backup = config.getBool(prefix + ".allow_backup", true);
|
||||
addReplicatedStorage(name, zookeeper_path, get_zookeeper_function, allow_backup);
|
||||
}
|
||||
else
|
||||
throw Exception("Unknown storage type '" + type + "' at " + prefix + " in config", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
|
||||
@ -383,12 +373,18 @@ void AccessControl::addStoragesFromMainConfig(
|
||||
if (users_config_path != config_path)
|
||||
checkForUsersNotInMainConfig(config, config_path, users_config_path, getLogger());
|
||||
|
||||
addUsersConfigStorage(users_config_path, include_from_path, dbms_dir, get_zookeeper_function);
|
||||
addUsersConfigStorage(
|
||||
UsersConfigAccessStorage::STORAGE_TYPE,
|
||||
users_config_path,
|
||||
include_from_path,
|
||||
dbms_dir,
|
||||
get_zookeeper_function,
|
||||
/* allow_backup= */ false);
|
||||
}
|
||||
|
||||
String disk_storage_dir = config.getString("access_control_path", "");
|
||||
if (!disk_storage_dir.empty())
|
||||
addDiskStorage(disk_storage_dir);
|
||||
addDiskStorage(DiskAccessStorage::STORAGE_TYPE, disk_storage_dir, /* readonly= */ false, /* allow_backup= */ true);
|
||||
|
||||
if (has_user_directories)
|
||||
addStoragesFromUserDirectoriesConfig(config, "user_directories", config_dir, dbms_dir, include_from_path, get_zookeeper_function);
|
||||
@ -463,6 +459,23 @@ UUID AccessControl::authenticate(const Credentials & credentials, const Poco::Ne
|
||||
}
|
||||
}
|
||||
|
||||
void AccessControl::backup(BackupEntriesCollector & backup_entries_collector, AccessEntityType type, const String & data_path_in_backup) const
|
||||
{
|
||||
backupAccessEntities(backup_entries_collector, data_path_in_backup, *this, type);
|
||||
}
|
||||
|
||||
void AccessControl::restore(RestorerFromBackup & restorer, const String & data_path_in_backup)
|
||||
{
|
||||
/// The restorer must already know about `data_path_in_backup`, but let's check.
|
||||
restorer.checkPathInBackupToRestoreAccess(data_path_in_backup);
|
||||
}
|
||||
|
||||
void AccessControl::insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr<IRestoreCoordination> restore_coordination)
|
||||
{
|
||||
MultipleAccessStorage::insertFromBackup(entities_from_backup, restore_settings, restore_coordination);
|
||||
changes_notifier->sendNotifications();
|
||||
}
|
||||
|
||||
void AccessControl::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config)
|
||||
{
|
||||
external_authenticators->setConfiguration(config, getLogger());
|
||||
|
||||
@ -42,6 +42,8 @@ class ClientInfo;
|
||||
class ExternalAuthenticators;
|
||||
class AccessChangesNotifier;
|
||||
struct Settings;
|
||||
class BackupEntriesCollector;
|
||||
class RestorerFromBackup;
|
||||
|
||||
|
||||
/// Manages access control entities.
|
||||
@ -60,37 +62,31 @@ public:
|
||||
void setUsersConfig(const Poco::Util::AbstractConfiguration & users_config_);
|
||||
|
||||
/// Adds UsersConfigAccessStorage.
|
||||
void addUsersConfigStorage(const Poco::Util::AbstractConfiguration & users_config_);
|
||||
|
||||
void addUsersConfigStorage(const String & storage_name_,
|
||||
const Poco::Util::AbstractConfiguration & users_config_);
|
||||
|
||||
void addUsersConfigStorage(const String & users_config_path_,
|
||||
const String & include_from_path_,
|
||||
const String & preprocessed_dir_,
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_ = {});
|
||||
const Poco::Util::AbstractConfiguration & users_config_,
|
||||
bool allow_backup_);
|
||||
|
||||
void addUsersConfigStorage(const String & storage_name_,
|
||||
const String & users_config_path_,
|
||||
const String & include_from_path_,
|
||||
const String & preprocessed_dir_,
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_ = {});
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_,
|
||||
bool allow_backup_);
|
||||
|
||||
/// Loads access entities from the directory on the local disk.
|
||||
/// Use that directory to keep created users/roles/etc.
|
||||
void addDiskStorage(const String & directory_, bool readonly_ = false);
|
||||
void addDiskStorage(const String & storage_name_, const String & directory_, bool readonly_ = false);
|
||||
void addDiskStorage(const String & storage_name_, const String & directory_, bool readonly_, bool allow_backup_);
|
||||
|
||||
/// Adds MemoryAccessStorage which keeps access entities in memory.
|
||||
void addMemoryStorage();
|
||||
void addMemoryStorage(const String & storage_name_);
|
||||
void addMemoryStorage(const String & storage_name_, bool allow_backup_);
|
||||
|
||||
/// Adds LDAPAccessStorage which allows querying remote LDAP server for user info.
|
||||
void addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_);
|
||||
|
||||
void addReplicatedStorage(const String & storage_name,
|
||||
const String & zookeeper_path,
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function);
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function,
|
||||
bool allow_backup);
|
||||
|
||||
/// Adds storages from <users_directories> config.
|
||||
void addStoragesFromUserDirectoriesConfig(const Poco::Util::AbstractConfiguration & config,
|
||||
@ -123,6 +119,11 @@ public:
|
||||
scope_guard subscribeForChanges(const std::vector<UUID> & ids, const OnChangedHandler & handler) const;
|
||||
|
||||
UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address) const;
|
||||
|
||||
/// Makes a backup of access entities.
|
||||
void backup(BackupEntriesCollector & backup_entries_collector, AccessEntityType type, const String & data_path_in_backup) const;
|
||||
static void restore(RestorerFromBackup & restorer, const String & data_path_in_backup);
|
||||
|
||||
void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config);
|
||||
|
||||
/// Sets the default profile's name.
|
||||
@ -197,6 +198,8 @@ public:
|
||||
/// Gets manager of notifications.
|
||||
AccessChangesNotifier & getChangesNotifier();
|
||||
|
||||
void insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr<IRestoreCoordination> restore_coordination) override;
|
||||
|
||||
private:
|
||||
class ContextAccessCache;
|
||||
class CustomSettingsPrefixes;
|
||||
|
||||
@ -736,6 +736,18 @@ AccessRights::AccessRights(const AccessFlags & access)
|
||||
}
|
||||
|
||||
|
||||
AccessRights::AccessRights(const AccessRightsElement & element)
|
||||
{
|
||||
grant(element);
|
||||
}
|
||||
|
||||
|
||||
AccessRights::AccessRights(const AccessRightsElements & elements)
|
||||
{
|
||||
grant(elements);
|
||||
}
|
||||
|
||||
|
||||
bool AccessRights::isEmpty() const
|
||||
{
|
||||
return !root && !root_with_grant_option;
|
||||
|
||||
@ -16,6 +16,9 @@ class AccessRights
|
||||
public:
|
||||
AccessRights();
|
||||
explicit AccessRights(const AccessFlags & access);
|
||||
explicit AccessRights(const AccessRightsElement & element);
|
||||
explicit AccessRights(const AccessRightsElements & elements);
|
||||
|
||||
~AccessRights();
|
||||
AccessRights(const AccessRights & src);
|
||||
AccessRights & operator =(const AccessRights & src);
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
#include <Access/Common/AccessEntityType.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <base/range.h>
|
||||
#include <boost/algorithm/string/case_conv.hpp>
|
||||
#include <boost/algorithm/string/predicate.hpp>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
|
||||
|
||||
@ -15,6 +17,7 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_QUOTA;
|
||||
extern const int THERE_IS_NO_PROFILE;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
|
||||
@ -83,4 +86,15 @@ const AccessEntityTypeInfo & AccessEntityTypeInfo::get(AccessEntityType type_)
|
||||
throw Exception("Unknown type: " + std::to_string(static_cast<size_t>(type_)), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
AccessEntityType AccessEntityTypeInfo::parseType(const String & name_)
|
||||
{
|
||||
for (auto type : collections::range(AccessEntityType::MAX))
|
||||
{
|
||||
const auto & info = get(type);
|
||||
if (boost::iequals(info.name, name_))
|
||||
return type;
|
||||
}
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown type: {}", name_);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -35,6 +35,7 @@ struct AccessEntityTypeInfo
|
||||
String formatEntityNameWithType(const String & entity_name) const;
|
||||
|
||||
static const AccessEntityTypeInfo & get(AccessEntityType type_);
|
||||
static AccessEntityType parseType(const String & name_);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -99,6 +99,7 @@ enum class AccessType
|
||||
\
|
||||
M(TRUNCATE, "TRUNCATE TABLE", TABLE, ALL) \
|
||||
M(OPTIMIZE, "OPTIMIZE TABLE", TABLE, ALL) \
|
||||
M(BACKUP, "", TABLE, ALL) /* allows to backup tables */\
|
||||
\
|
||||
M(KILL_QUERY, "", GLOBAL, ALL) /* allows to kill a query started by another user
|
||||
(anyone can kill his own queries) */\
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#include <Access/DiskAccessStorage.h>
|
||||
#include <Access/AccessEntityIO.h>
|
||||
#include <Access/AccessChangesNotifier.h>
|
||||
#include <Backups/RestoreSettings.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
@ -165,11 +166,12 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
DiskAccessStorage::DiskAccessStorage(const String & storage_name_, const String & directory_path_, bool readonly_, AccessChangesNotifier & changes_notifier_)
|
||||
DiskAccessStorage::DiskAccessStorage(const String & storage_name_, const String & directory_path_, AccessChangesNotifier & changes_notifier_, bool readonly_, bool allow_backup_)
|
||||
: IAccessStorage(storage_name_), changes_notifier(changes_notifier_)
|
||||
{
|
||||
directory_path = makeDirectoryPathCanonical(directory_path_);
|
||||
readonly = readonly_;
|
||||
backup_allowed = allow_backup_;
|
||||
|
||||
std::error_code create_dir_error_code;
|
||||
std::filesystem::create_directories(directory_path, create_dir_error_code);
|
||||
@ -457,7 +459,7 @@ AccessEntityPtr DiskAccessStorage::readImpl(const UUID & id, bool throw_if_not_e
|
||||
}
|
||||
|
||||
|
||||
std::optional<String> DiskAccessStorage::readNameImpl(const UUID & id, bool throw_if_not_exists) const
|
||||
std::optional<std::pair<String, AccessEntityType>> DiskAccessStorage::readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const
|
||||
{
|
||||
std::lock_guard lock{mutex};
|
||||
auto it = entries_by_id.find(id);
|
||||
@ -468,21 +470,27 @@ std::optional<String> DiskAccessStorage::readNameImpl(const UUID & id, bool thro
|
||||
else
|
||||
return std::nullopt;
|
||||
}
|
||||
return it->second.name;
|
||||
return std::make_pair(it->second.name, it->second.type);
|
||||
}
|
||||
|
||||
|
||||
std::optional<UUID> DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
||||
{
|
||||
UUID id = generateRandomID();
|
||||
std::lock_guard lock{mutex};
|
||||
if (insertNoLock(id, new_entity, replace_if_exists, throw_if_exists))
|
||||
if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists))
|
||||
return id;
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
|
||||
bool DiskAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
||||
{
|
||||
std::lock_guard lock{mutex};
|
||||
return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists);
|
||||
}
|
||||
|
||||
|
||||
bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
||||
{
|
||||
const String & name = new_entity->getName();
|
||||
@ -649,4 +657,20 @@ void DiskAccessStorage::deleteAccessEntityOnDisk(const UUID & id) const
|
||||
throw Exception("Couldn't delete " + file_path, ErrorCodes::FILE_DOESNT_EXIST);
|
||||
}
|
||||
|
||||
|
||||
void DiskAccessStorage::insertFromBackup(
|
||||
const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup,
|
||||
const RestoreSettings & restore_settings,
|
||||
std::shared_ptr<IRestoreCoordination>)
|
||||
{
|
||||
if (!isRestoreAllowed())
|
||||
throwRestoreNotAllowed();
|
||||
|
||||
bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace);
|
||||
bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate);
|
||||
|
||||
for (const auto & [id, entity] : entities_from_backup)
|
||||
insertWithID(id, entity, replace_if_exists, throw_if_exists);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -15,7 +15,7 @@ class DiskAccessStorage : public IAccessStorage
|
||||
public:
|
||||
static constexpr char STORAGE_TYPE[] = "local directory";
|
||||
|
||||
DiskAccessStorage(const String & storage_name_, const String & directory_path_, bool readonly_, AccessChangesNotifier & changes_notifier_);
|
||||
DiskAccessStorage(const String & storage_name_, const String & directory_path_, AccessChangesNotifier & changes_notifier_, bool readonly_, bool allow_backup_);
|
||||
~DiskAccessStorage() override;
|
||||
|
||||
const char * getStorageType() const override { return STORAGE_TYPE; }
|
||||
@ -29,11 +29,14 @@ public:
|
||||
|
||||
bool exists(const UUID & id) const override;
|
||||
|
||||
bool isBackupAllowed() const override { return backup_allowed; }
|
||||
void insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr<IRestoreCoordination> restore_coordination) override;
|
||||
|
||||
private:
|
||||
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
||||
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
std::optional<String> readNameImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
|
||||
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
||||
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
||||
@ -47,6 +50,7 @@ private:
|
||||
void listsWritingThreadFunc();
|
||||
void stopListsWritingThread();
|
||||
|
||||
bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists);
|
||||
bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists);
|
||||
bool removeNoLock(const UUID & id, bool throw_if_not_exists);
|
||||
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
||||
@ -65,7 +69,6 @@ private:
|
||||
};
|
||||
|
||||
String directory_path;
|
||||
std::atomic<bool> readonly;
|
||||
std::unordered_map<UUID, Entry> entries_by_id;
|
||||
std::unordered_map<std::string_view, Entry *> entries_by_name_and_type[static_cast<size_t>(AccessEntityType::MAX)];
|
||||
boost::container::flat_set<AccessEntityType> types_of_lists_to_write;
|
||||
@ -74,6 +77,8 @@ private:
|
||||
std::condition_variable lists_writing_thread_should_exit; /// Signals `lists_writing_thread` to exit.
|
||||
bool lists_writing_thread_is_waiting = false;
|
||||
AccessChangesNotifier & changes_notifier;
|
||||
std::atomic<bool> readonly;
|
||||
std::atomic<bool> backup_allowed;
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/SettingsProfileElement.h>
|
||||
#include <base/defines.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <mutex>
|
||||
@ -42,7 +43,7 @@ private:
|
||||
void setInfo(const std::shared_ptr<const SettingsProfilesInfo> & info_);
|
||||
|
||||
const Params params;
|
||||
std::shared_ptr<const SettingsProfilesInfo> info;
|
||||
std::shared_ptr<const SettingsProfilesInfo> info TSA_GUARDED_BY(mutex);
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
}
|
||||
|
||||
@ -231,18 +231,23 @@ void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco
|
||||
params.prefix = config.getString(prefix + ".prefix");
|
||||
}
|
||||
|
||||
void ExternalAuthenticators::reset()
|
||||
void ExternalAuthenticators::resetImpl()
|
||||
{
|
||||
std::scoped_lock lock(mutex);
|
||||
ldap_client_params_blueprint.clear();
|
||||
ldap_caches.clear();
|
||||
kerberos_params.reset();
|
||||
}
|
||||
|
||||
void ExternalAuthenticators::reset()
|
||||
{
|
||||
std::scoped_lock lock(mutex);
|
||||
resetImpl();
|
||||
}
|
||||
|
||||
void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
|
||||
{
|
||||
std::scoped_lock lock(mutex);
|
||||
reset();
|
||||
resetImpl();
|
||||
|
||||
Poco::Util::AbstractConfiguration::Keys all_keys;
|
||||
config.keys("", all_keys);
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Access/LDAPClient.h>
|
||||
#include <Access/Credentials.h>
|
||||
#include <Access/GSSAcceptor.h>
|
||||
#include <base/defines.h>
|
||||
#include <base/types.h>
|
||||
|
||||
#include <chrono>
|
||||
@ -22,7 +23,6 @@ namespace Poco
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -51,10 +51,12 @@ private:
|
||||
using LDAPCaches = std::map<String, LDAPCache>; // server name -> cache
|
||||
using LDAPParams = std::map<String, LDAPClient::Params>; // server name -> params
|
||||
|
||||
mutable std::recursive_mutex mutex;
|
||||
LDAPParams ldap_client_params_blueprint;
|
||||
mutable LDAPCaches ldap_caches;
|
||||
std::optional<GSSAcceptorContext::Params> kerberos_params;
|
||||
mutable std::mutex mutex;
|
||||
LDAPParams ldap_client_params_blueprint TSA_GUARDED_BY(mutex) ;
|
||||
mutable LDAPCaches ldap_caches TSA_GUARDED_BY(mutex) ;
|
||||
std::optional<GSSAcceptorContext::Params> kerberos_params TSA_GUARDED_BY(mutex) ;
|
||||
|
||||
void resetImpl() TSA_REQUIRES(mutex);
|
||||
};
|
||||
|
||||
void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix);
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
#include <Access/RolesOrUsersSet.h>
|
||||
#include <boost/range/algorithm/set_algorithm.hpp>
|
||||
#include <boost/range/algorithm_ext/erase.hpp>
|
||||
#include <boost/range/algorithm/copy.hpp>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -166,4 +168,57 @@ void GrantedRoles::makeIntersection(const GrantedRoles & other)
|
||||
return other.roles_with_admin_option.find(id) == other.roles_with_admin_option.end();
|
||||
});
|
||||
}
|
||||
|
||||
std::vector<UUID> GrantedRoles::findDependencies() const
|
||||
{
|
||||
std::vector<UUID> res;
|
||||
boost::range::copy(roles, std::back_inserter(res));
|
||||
return res;
|
||||
}
|
||||
|
||||
void GrantedRoles::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||
{
|
||||
std::vector<UUID> new_ids;
|
||||
|
||||
for (auto it = roles.begin(); it != roles.end();)
|
||||
{
|
||||
auto id = *it;
|
||||
auto it_new_id = old_to_new_ids.find(id);
|
||||
if (it_new_id != old_to_new_ids.end())
|
||||
{
|
||||
auto new_id = it_new_id->second;
|
||||
new_ids.push_back(new_id);
|
||||
it = roles.erase(it);
|
||||
}
|
||||
else
|
||||
{
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
if (!new_ids.empty())
|
||||
{
|
||||
boost::range::copy(new_ids, std::inserter(roles, roles.end()));
|
||||
new_ids.clear();
|
||||
|
||||
for (auto it = roles_with_admin_option.begin(); it != roles_with_admin_option.end();)
|
||||
{
|
||||
auto id = *it;
|
||||
auto it_new_id = old_to_new_ids.find(id);
|
||||
if (it_new_id != old_to_new_ids.end())
|
||||
{
|
||||
auto new_id = it_new_id->second;
|
||||
new_ids.push_back(new_id);
|
||||
it = roles_with_admin_option.erase(it);
|
||||
}
|
||||
else
|
||||
{
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
boost::range::copy(new_ids, std::inserter(roles_with_admin_option, roles_with_admin_option.end()));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Core/UUID.h>
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <vector>
|
||||
#include <unordered_map>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -24,6 +25,8 @@ public:
|
||||
void revokeAdminOption(const UUID & role_);
|
||||
void revokeAdminOption(const std::vector<UUID> & roles_);
|
||||
|
||||
bool isEmpty() const { return roles.empty(); }
|
||||
|
||||
bool isGranted(const UUID & role_) const;
|
||||
bool isGrantedWithAdminOption(const UUID & role_) const;
|
||||
|
||||
@ -54,6 +57,9 @@ public:
|
||||
friend bool operator ==(const GrantedRoles & left, const GrantedRoles & right) { return (left.roles == right.roles) && (left.roles_with_admin_option == right.roles_with_admin_option); }
|
||||
friend bool operator !=(const GrantedRoles & left, const GrantedRoles & right) { return !(left == right); }
|
||||
|
||||
std::vector<UUID> findDependencies() const;
|
||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
|
||||
|
||||
private:
|
||||
boost::container::flat_set<UUID> roles;
|
||||
boost::container::flat_set<UUID> roles_with_admin_option;
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <base/types.h>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -45,6 +46,15 @@ struct IAccessEntity
|
||||
bool operator()(const std::shared_ptr<const IAccessEntity> & lhs, const std::shared_ptr<const IAccessEntity> & rhs) const { return operator()(*lhs, *rhs); }
|
||||
};
|
||||
|
||||
/// Finds all dependencies.
|
||||
virtual std::vector<UUID> findDependencies() const { return {}; }
|
||||
|
||||
/// Replaces dependencies according to a specified map.
|
||||
virtual void replaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
|
||||
|
||||
/// Whether this access entity should be written to a backup.
|
||||
virtual bool isBackupAllowed() const { return false; }
|
||||
|
||||
protected:
|
||||
String name;
|
||||
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
#include <base/FnTraits.h>
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
#include <boost/range/algorithm_ext/erase.hpp>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -19,6 +20,7 @@ namespace ErrorCodes
|
||||
extern const int ACCESS_ENTITY_ALREADY_EXISTS;
|
||||
extern const int ACCESS_ENTITY_NOT_FOUND;
|
||||
extern const int ACCESS_STORAGE_READONLY;
|
||||
extern const int ACCESS_STORAGE_DOESNT_ALLOW_BACKUP;
|
||||
extern const int WRONG_PASSWORD;
|
||||
extern const int IP_ADDRESS_NOT_ALLOWED;
|
||||
extern const int LOGICAL_ERROR;
|
||||
@ -83,13 +85,15 @@ std::vector<UUID> IAccessStorage::getIDs(AccessEntityType type, const Strings &
|
||||
|
||||
String IAccessStorage::readName(const UUID & id) const
|
||||
{
|
||||
return *readNameImpl(id, /* throw_if_not_exists = */ true);
|
||||
return readNameWithType(id).first;
|
||||
}
|
||||
|
||||
|
||||
std::optional<String> IAccessStorage::readName(const UUID & id, bool throw_if_not_exists) const
|
||||
{
|
||||
return readNameImpl(id, throw_if_not_exists);
|
||||
if (auto name_and_type = readNameWithType(id, throw_if_not_exists))
|
||||
return name_and_type->first;
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
|
||||
@ -99,7 +103,7 @@ Strings IAccessStorage::readNames(const std::vector<UUID> & ids, bool throw_if_n
|
||||
res.reserve(ids.size());
|
||||
for (const auto & id : ids)
|
||||
{
|
||||
if (auto name = readNameImpl(id, throw_if_not_exists))
|
||||
if (auto name = readName(id, throw_if_not_exists))
|
||||
res.emplace_back(std::move(name).value());
|
||||
}
|
||||
return res;
|
||||
@ -118,14 +122,42 @@ Strings IAccessStorage::tryReadNames(const std::vector<UUID> & ids) const
|
||||
}
|
||||
|
||||
|
||||
std::optional<String> IAccessStorage::readNameImpl(const UUID & id, bool throw_if_not_exists) const
|
||||
std::pair<String, AccessEntityType> IAccessStorage::readNameWithType(const UUID & id) const
|
||||
{
|
||||
return *readNameWithTypeImpl(id, /* throw_if_not_exists = */ true);
|
||||
}
|
||||
|
||||
std::optional<std::pair<String, AccessEntityType>> IAccessStorage::readNameWithType(const UUID & id, bool throw_if_not_exists) const
|
||||
{
|
||||
return readNameWithTypeImpl(id, throw_if_not_exists);
|
||||
}
|
||||
|
||||
std::optional<std::pair<String, AccessEntityType>> IAccessStorage::tryReadNameWithType(const UUID & id) const
|
||||
{
|
||||
return readNameWithTypeImpl(id, /* throw_if_not_exists = */ false);
|
||||
}
|
||||
|
||||
|
||||
std::optional<std::pair<String, AccessEntityType>> IAccessStorage::readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const
|
||||
{
|
||||
if (auto entity = read(id, throw_if_not_exists))
|
||||
return entity->getName();
|
||||
return std::make_pair(entity->getName(), entity->getType());
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
|
||||
std::vector<std::pair<UUID, AccessEntityPtr>> IAccessStorage::readAllWithIDs(AccessEntityType type) const
|
||||
{
|
||||
std::vector<std::pair<UUID, AccessEntityPtr>> entities;
|
||||
for (const auto & id : findAll(type))
|
||||
{
|
||||
if (auto entity = tryRead(id))
|
||||
entities.emplace_back(id, entity);
|
||||
}
|
||||
return entities;
|
||||
}
|
||||
|
||||
|
||||
UUID IAccessStorage::insert(const AccessEntityPtr & entity)
|
||||
{
|
||||
return *insert(entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true);
|
||||
@ -488,6 +520,29 @@ bool IAccessStorage::isAddressAllowed(const User & user, const Poco::Net::IPAddr
|
||||
}
|
||||
|
||||
|
||||
bool IAccessStorage::isRestoreAllowed() const
|
||||
{
|
||||
return isBackupAllowed() && !isReadOnly();
|
||||
}
|
||||
|
||||
std::vector<std::pair<UUID, AccessEntityPtr>> IAccessStorage::readAllForBackup(AccessEntityType type, const BackupSettings &) const
|
||||
{
|
||||
if (!isBackupAllowed())
|
||||
throwBackupNotAllowed();
|
||||
|
||||
auto res = readAllWithIDs(type);
|
||||
boost::range::remove_erase_if(res, [](const std::pair<UUID, AccessEntityPtr> & x) { return !x.second->isBackupAllowed(); });
|
||||
return res;
|
||||
}
|
||||
|
||||
void IAccessStorage::insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> &, const RestoreSettings &, std::shared_ptr<IRestoreCoordination>)
|
||||
{
|
||||
if (!isRestoreAllowed())
|
||||
throwRestoreNotAllowed();
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "insertFromBackup() is not implemented in {}", getStorageType());
|
||||
}
|
||||
|
||||
|
||||
UUID IAccessStorage::generateRandomID()
|
||||
{
|
||||
static Poco::UUIDGenerator generator;
|
||||
@ -577,6 +632,7 @@ void IAccessStorage::throwReadonlyCannotRemove(AccessEntityType type, const Stri
|
||||
ErrorCodes::ACCESS_STORAGE_READONLY);
|
||||
}
|
||||
|
||||
|
||||
void IAccessStorage::throwAddressNotAllowed(const Poco::Net::IPAddress & address)
|
||||
{
|
||||
throw Exception("Connections from " + address.toString() + " are not allowed", ErrorCodes::IP_ADDRESS_NOT_ALLOWED);
|
||||
@ -589,9 +645,20 @@ void IAccessStorage::throwAuthenticationTypeNotAllowed(AuthenticationType auth_t
|
||||
"Authentication type {} is not allowed, check the setting allow_{} in the server configuration",
|
||||
toString(auth_type), AuthenticationTypeInfo::get(auth_type).name);
|
||||
}
|
||||
|
||||
void IAccessStorage::throwInvalidCredentials()
|
||||
{
|
||||
throw Exception("Invalid credentials", ErrorCodes::WRONG_PASSWORD);
|
||||
}
|
||||
|
||||
void IAccessStorage::throwBackupNotAllowed() const
|
||||
{
|
||||
throw Exception(ErrorCodes::ACCESS_STORAGE_DOESNT_ALLOW_BACKUP, "Backup of access entities is not allowed in {}", getStorageName());
|
||||
}
|
||||
|
||||
void IAccessStorage::throwRestoreNotAllowed() const
|
||||
{
|
||||
throw Exception(ErrorCodes::ACCESS_STORAGE_DOESNT_ALLOW_BACKUP, "Restore of access entities is not allowed in {}", getStorageName());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -18,6 +18,9 @@ struct User;
|
||||
class Credentials;
|
||||
class ExternalAuthenticators;
|
||||
enum class AuthenticationType;
|
||||
struct BackupSettings;
|
||||
struct RestoreSettings;
|
||||
class IRestoreCoordination;
|
||||
|
||||
/// Contains entities, i.e. instances of classes derived from IAccessEntity.
|
||||
/// The implementations of this class MUST be thread-safe.
|
||||
@ -101,6 +104,16 @@ public:
|
||||
std::optional<String> tryReadName(const UUID & id) const;
|
||||
Strings tryReadNames(const std::vector<UUID> & ids) const;
|
||||
|
||||
std::pair<String, AccessEntityType> readNameWithType(const UUID & id) const;
|
||||
std::optional<std::pair<String, AccessEntityType>> readNameWithType(const UUID & id, bool throw_if_not_exists) const;
|
||||
std::optional<std::pair<String, AccessEntityType>> tryReadNameWithType(const UUID & id) const;
|
||||
|
||||
/// Reads all entities and returns them with their IDs.
|
||||
template <typename EntityClassT>
|
||||
std::vector<std::pair<UUID, std::shared_ptr<const EntityClassT>>> readAllWithIDs() const;
|
||||
|
||||
std::vector<std::pair<UUID, AccessEntityPtr>> readAllWithIDs(AccessEntityType type) const;
|
||||
|
||||
/// Inserts an entity to the storage. Returns ID of a new entry in the storage.
|
||||
/// Throws an exception if the specified name already exists.
|
||||
UUID insert(const AccessEntityPtr & entity);
|
||||
@ -143,11 +156,19 @@ public:
|
||||
UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool allow_no_password, bool allow_plaintext_password) const;
|
||||
std::optional<UUID> authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const;
|
||||
|
||||
/// Returns true if this storage can be stored to or restored from a backup.
|
||||
virtual bool isBackupAllowed() const { return false; }
|
||||
virtual bool isRestoreAllowed() const;
|
||||
|
||||
/// Makes a backup of this access storage.
|
||||
virtual std::vector<std::pair<UUID, AccessEntityPtr>> readAllForBackup(AccessEntityType type, const BackupSettings & backup_settings) const;
|
||||
virtual void insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr<IRestoreCoordination> restore_coordination);
|
||||
|
||||
protected:
|
||||
virtual std::optional<UUID> findImpl(AccessEntityType type, const String & name) const = 0;
|
||||
virtual std::vector<UUID> findAllImpl(AccessEntityType type) const = 0;
|
||||
virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0;
|
||||
virtual std::optional<String> readNameImpl(const UUID & id, bool throw_if_not_exists) const;
|
||||
virtual std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const;
|
||||
virtual std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
|
||||
virtual bool removeImpl(const UUID & id, bool throw_if_not_exists);
|
||||
virtual bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
||||
@ -170,6 +191,8 @@ protected:
|
||||
[[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address);
|
||||
[[noreturn]] static void throwInvalidCredentials();
|
||||
[[noreturn]] static void throwAuthenticationTypeNotAllowed(AuthenticationType auth_type);
|
||||
[[noreturn]] void throwBackupNotAllowed() const;
|
||||
[[noreturn]] void throwRestoreNotAllowed() const;
|
||||
|
||||
private:
|
||||
const String storage_name;
|
||||
@ -218,4 +241,17 @@ std::shared_ptr<const EntityClassT> IAccessStorage::tryRead(const String & name)
|
||||
{
|
||||
return read<EntityClassT>(name, false);
|
||||
}
|
||||
|
||||
template <typename EntityClassT>
|
||||
std::vector<std::pair<UUID, std::shared_ptr<const EntityClassT>>> IAccessStorage::readAllWithIDs() const
|
||||
{
|
||||
std::vector<std::pair<UUID, std::shared_ptr<const EntityClassT>>> entities;
|
||||
for (const auto & id : findAll<EntityClassT>())
|
||||
{
|
||||
if (auto entity = tryRead<EntityClassT>(id))
|
||||
entities.emplace_back(id, entity);
|
||||
}
|
||||
return entities;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -28,7 +28,7 @@ namespace ErrorCodes
|
||||
|
||||
|
||||
LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix)
|
||||
: IAccessStorage(storage_name_), access_control(access_control_), memory_storage(storage_name_, access_control.getChangesNotifier())
|
||||
: IAccessStorage(storage_name_), access_control(access_control_), memory_storage(storage_name_, access_control.getChangesNotifier(), false)
|
||||
{
|
||||
setConfiguration(config, prefix);
|
||||
}
|
||||
@ -36,6 +36,7 @@ LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControl
|
||||
|
||||
String LDAPAccessStorage::getLDAPServerName() const
|
||||
{
|
||||
std::scoped_lock lock(mutex);
|
||||
return ldap_server_name;
|
||||
}
|
||||
|
||||
@ -442,10 +443,10 @@ AccessEntityPtr LDAPAccessStorage::readImpl(const UUID & id, bool throw_if_not_e
|
||||
}
|
||||
|
||||
|
||||
std::optional<String> LDAPAccessStorage::readNameImpl(const UUID & id, bool throw_if_not_exists) const
|
||||
std::optional<std::pair<String, AccessEntityType>> LDAPAccessStorage::readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const
|
||||
{
|
||||
std::scoped_lock lock(mutex);
|
||||
return memory_storage.readName(id, throw_if_not_exists);
|
||||
return memory_storage.readNameWithType(id, throw_if_not_exists);
|
||||
}
|
||||
|
||||
|
||||
@ -504,4 +505,5 @@ std::optional<UUID> LDAPAccessStorage::authenticateImpl(
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -47,7 +47,7 @@ private: // IAccessStorage implementations.
|
||||
virtual std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
||||
virtual std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||
virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
virtual std::optional<String> readNameImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
virtual std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
virtual std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override;
|
||||
|
||||
void setConfiguration(const Poco::Util::AbstractConfiguration & config, const String & prefix);
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <Access/MemoryAccessStorage.h>
|
||||
#include <Access/AccessChangesNotifier.h>
|
||||
#include <Backups/RestoreSettings.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <boost/range/adaptor/map.hpp>
|
||||
@ -8,8 +9,8 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
MemoryAccessStorage::MemoryAccessStorage(const String & storage_name_, AccessChangesNotifier & changes_notifier_)
|
||||
: IAccessStorage(storage_name_), changes_notifier(changes_notifier_)
|
||||
MemoryAccessStorage::MemoryAccessStorage(const String & storage_name_, AccessChangesNotifier & changes_notifier_, bool allow_backup_)
|
||||
: IAccessStorage(storage_name_), changes_notifier(changes_notifier_), backup_allowed(allow_backup_)
|
||||
{
|
||||
}
|
||||
|
||||
@ -65,14 +66,20 @@ AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id, bool throw_if_not
|
||||
std::optional<UUID> MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
||||
{
|
||||
UUID id = generateRandomID();
|
||||
std::lock_guard lock{mutex};
|
||||
if (insertNoLock(id, new_entity, replace_if_exists, throw_if_exists))
|
||||
if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists))
|
||||
return id;
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
|
||||
bool MemoryAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
||||
{
|
||||
std::lock_guard lock{mutex};
|
||||
return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists);
|
||||
}
|
||||
|
||||
|
||||
bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
||||
{
|
||||
const String & name = new_entity->getName();
|
||||
@ -264,4 +271,20 @@ void MemoryAccessStorage::setAll(const std::vector<std::pair<UUID, AccessEntityP
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void MemoryAccessStorage::insertFromBackup(
|
||||
const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup,
|
||||
const RestoreSettings & restore_settings,
|
||||
std::shared_ptr<IRestoreCoordination>)
|
||||
{
|
||||
if (!isRestoreAllowed())
|
||||
throwRestoreNotAllowed();
|
||||
|
||||
bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace);
|
||||
bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate);
|
||||
|
||||
for (const auto & [id, entity] : entities_from_backup)
|
||||
insertWithID(id, entity, replace_if_exists, throw_if_exists);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/IAccessStorage.h>
|
||||
#include <base/defines.h>
|
||||
#include <list>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
@ -17,7 +18,7 @@ class MemoryAccessStorage : public IAccessStorage
|
||||
public:
|
||||
static constexpr char STORAGE_TYPE[] = "memory";
|
||||
|
||||
explicit MemoryAccessStorage(const String & storage_name_, AccessChangesNotifier & changes_notifier_);
|
||||
explicit MemoryAccessStorage(const String & storage_name_, AccessChangesNotifier & changes_notifier_, bool allow_backup_);
|
||||
|
||||
const char * getStorageType() const override { return STORAGE_TYPE; }
|
||||
|
||||
@ -27,6 +28,9 @@ public:
|
||||
|
||||
bool exists(const UUID & id) const override;
|
||||
|
||||
bool isBackupAllowed() const override { return backup_allowed; }
|
||||
void insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr<IRestoreCoordination> restore_coordination) override;
|
||||
|
||||
private:
|
||||
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
||||
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||
@ -35,9 +39,10 @@ private:
|
||||
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
||||
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
||||
|
||||
bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
|
||||
bool removeNoLock(const UUID & id, bool throw_if_not_exists);
|
||||
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
||||
bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists);
|
||||
bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) TSA_REQUIRES(mutex);
|
||||
bool removeNoLock(const UUID & id, bool throw_if_not_exists) TSA_REQUIRES(mutex);
|
||||
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) TSA_REQUIRES(mutex);
|
||||
|
||||
struct Entry
|
||||
{
|
||||
@ -46,8 +51,9 @@ private:
|
||||
};
|
||||
|
||||
mutable std::mutex mutex;
|
||||
std::unordered_map<UUID, Entry> entries_by_id; /// We want to search entries both by ID and by the pair of name and type.
|
||||
std::unordered_map<String, Entry *> entries_by_name_and_type[static_cast<size_t>(AccessEntityType::MAX)];
|
||||
std::unordered_map<UUID, Entry> entries_by_id TSA_GUARDED_BY(mutex); /// We want to search entries both by ID and by the pair of name and type.
|
||||
std::unordered_map<String, Entry *> entries_by_name_and_type[static_cast<size_t>(AccessEntityType::MAX)] TSA_GUARDED_BY(mutex);
|
||||
AccessChangesNotifier & changes_notifier;
|
||||
bool backup_allowed = false;
|
||||
};
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <base/range.h>
|
||||
#include <base/insertAtEnd.h>
|
||||
#include <boost/range/adaptor/map.hpp>
|
||||
#include <boost/range/adaptor/reversed.hpp>
|
||||
#include <boost/range/algorithm/copy.hpp>
|
||||
@ -42,14 +43,14 @@ MultipleAccessStorage::~MultipleAccessStorage()
|
||||
|
||||
void MultipleAccessStorage::setStorages(const std::vector<StoragePtr> & storages)
|
||||
{
|
||||
std::unique_lock lock{mutex};
|
||||
std::lock_guard lock{mutex};
|
||||
nested_storages = std::make_shared<const Storages>(storages);
|
||||
ids_cache.reset();
|
||||
}
|
||||
|
||||
void MultipleAccessStorage::addStorage(const StoragePtr & new_storage)
|
||||
{
|
||||
std::unique_lock lock{mutex};
|
||||
std::lock_guard lock{mutex};
|
||||
if (boost::range::find(*nested_storages, new_storage) != nested_storages->end())
|
||||
return;
|
||||
auto new_storages = std::make_shared<Storages>(*nested_storages);
|
||||
@ -59,7 +60,7 @@ void MultipleAccessStorage::addStorage(const StoragePtr & new_storage)
|
||||
|
||||
void MultipleAccessStorage::removeStorage(const StoragePtr & storage_to_remove)
|
||||
{
|
||||
std::unique_lock lock{mutex};
|
||||
std::lock_guard lock{mutex};
|
||||
auto it = boost::range::find(*nested_storages, storage_to_remove);
|
||||
if (it == nested_storages->end())
|
||||
return;
|
||||
@ -189,10 +190,10 @@ AccessEntityPtr MultipleAccessStorage::readImpl(const UUID & id, bool throw_if_n
|
||||
}
|
||||
|
||||
|
||||
std::optional<String> MultipleAccessStorage::readNameImpl(const UUID & id, bool throw_if_not_exists) const
|
||||
std::optional<std::pair<String, AccessEntityType>> MultipleAccessStorage::readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const
|
||||
{
|
||||
if (auto storage = findStorage(id))
|
||||
return storage->readName(id, throw_if_not_exists);
|
||||
return storage->readNameWithType(id, throw_if_not_exists);
|
||||
|
||||
if (throw_if_not_exists)
|
||||
throwNotFound(id);
|
||||
@ -357,4 +358,65 @@ MultipleAccessStorage::authenticateImpl(const Credentials & credentials, const P
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
|
||||
bool MultipleAccessStorage::isBackupAllowed() const
|
||||
{
|
||||
auto storages = getStoragesInternal();
|
||||
for (const auto & storage : *storages)
|
||||
{
|
||||
if (storage->isBackupAllowed())
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
bool MultipleAccessStorage::isRestoreAllowed() const
|
||||
{
|
||||
auto storages = getStoragesInternal();
|
||||
for (const auto & storage : *storages)
|
||||
{
|
||||
if (storage->isRestoreAllowed())
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
std::vector<std::pair<UUID, AccessEntityPtr>> MultipleAccessStorage::readAllForBackup(AccessEntityType type, const BackupSettings & backup_settings) const
|
||||
{
|
||||
std::vector<std::pair<UUID, AccessEntityPtr>> res;
|
||||
auto storages = getStoragesInternal();
|
||||
size_t count = 0;
|
||||
|
||||
for (const auto & storage : *storages)
|
||||
{
|
||||
if (storage->isBackupAllowed())
|
||||
{
|
||||
insertAtEnd(res, storage->readAllForBackup(type, backup_settings));
|
||||
++count;
|
||||
}
|
||||
}
|
||||
|
||||
if (!count)
|
||||
throwBackupNotAllowed();
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
void MultipleAccessStorage::insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr<IRestoreCoordination> restore_coordination)
|
||||
{
|
||||
auto storages = getStoragesInternal();
|
||||
for (const auto & storage : *storages)
|
||||
{
|
||||
if (storage->isRestoreAllowed())
|
||||
{
|
||||
storage->insertFromBackup(entities_from_backup, restore_settings, restore_coordination);
|
||||
return;
|
||||
}
|
||||
}
|
||||
throwRestoreNotAllowed();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/IAccessStorage.h>
|
||||
#include <base/defines.h>
|
||||
#include <Common/LRUCache.h>
|
||||
#include <mutex>
|
||||
|
||||
@ -42,11 +43,16 @@ public:
|
||||
|
||||
bool exists(const UUID & id) const override;
|
||||
|
||||
bool isBackupAllowed() const override;
|
||||
bool isRestoreAllowed() const override;
|
||||
std::vector<std::pair<UUID, AccessEntityPtr>> readAllForBackup(AccessEntityType type, const BackupSettings & backup_settings) const override;
|
||||
void insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr<IRestoreCoordination> restore_coordination) override;
|
||||
|
||||
protected:
|
||||
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
||||
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
std::optional<String> readNameImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
|
||||
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
||||
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
||||
@ -56,8 +62,8 @@ private:
|
||||
using Storages = std::vector<StoragePtr>;
|
||||
std::shared_ptr<const Storages> getStoragesInternal() const;
|
||||
|
||||
std::shared_ptr<const Storages> nested_storages;
|
||||
mutable LRUCache<UUID, Storage> ids_cache;
|
||||
std::shared_ptr<const Storages> nested_storages TSA_GUARDED_BY(mutex);
|
||||
mutable LRUCache<UUID, Storage> ids_cache TSA_GUARDED_BY(mutex);
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
|
||||
|
||||
@ -19,5 +19,14 @@ bool Quota::equal(const IAccessEntity & other) const
|
||||
return (all_limits == other_quota.all_limits) && (key_type == other_quota.key_type) && (to_roles == other_quota.to_roles);
|
||||
}
|
||||
|
||||
std::vector<UUID> Quota::findDependencies() const
|
||||
{
|
||||
return to_roles.findDependencies();
|
||||
}
|
||||
|
||||
void Quota::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||
{
|
||||
to_roles.replaceDependencies(old_to_new_ids);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -45,6 +45,10 @@ struct Quota : public IAccessEntity
|
||||
std::shared_ptr<IAccessEntity> clone() const override { return cloneImpl<Quota>(); }
|
||||
static constexpr const auto TYPE = AccessEntityType::QUOTA;
|
||||
AccessEntityType getType() const override { return TYPE; }
|
||||
|
||||
std::vector<UUID> findDependencies() const override;
|
||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
||||
bool isBackupAllowed() const override { return true; }
|
||||
};
|
||||
|
||||
using QuotaPtr = std::shared_ptr<const Quota>;
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
#include <Access/MemoryAccessStorage.h>
|
||||
#include <Access/ReplicatedAccessStorage.h>
|
||||
#include <Access/AccessChangesNotifier.h>
|
||||
#include <Backups/RestoreSettings.h>
|
||||
#include <Backups/IRestoreCoordination.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <Common/ZooKeeper/KeeperException.h>
|
||||
@ -33,12 +35,14 @@ ReplicatedAccessStorage::ReplicatedAccessStorage(
|
||||
const String & storage_name_,
|
||||
const String & zookeeper_path_,
|
||||
zkutil::GetZooKeeper get_zookeeper_,
|
||||
AccessChangesNotifier & changes_notifier_)
|
||||
AccessChangesNotifier & changes_notifier_,
|
||||
bool allow_backup_)
|
||||
: IAccessStorage(storage_name_)
|
||||
, zookeeper_path(zookeeper_path_)
|
||||
, get_zookeeper(get_zookeeper_)
|
||||
, watched_queue(std::make_shared<ConcurrentBoundedQueue<UUID>>(std::numeric_limits<size_t>::max()))
|
||||
, changes_notifier(changes_notifier_)
|
||||
, backup_allowed(allow_backup_)
|
||||
{
|
||||
if (zookeeper_path.empty())
|
||||
throw Exception("ZooKeeper path must be non-empty", ErrorCodes::BAD_ARGUMENTS);
|
||||
@ -99,6 +103,15 @@ static void retryOnZooKeeperUserError(size_t attempts, Func && function)
|
||||
std::optional<UUID> ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
||||
{
|
||||
const UUID id = generateRandomID();
|
||||
if (insertWithID(id, new_entity, replace_if_exists, throw_if_exists))
|
||||
return id;
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
|
||||
bool ReplicatedAccessStorage::insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
||||
{
|
||||
const AccessEntityTypeInfo type_info = AccessEntityTypeInfo::get(new_entity->getType());
|
||||
const String & name = new_entity->getName();
|
||||
LOG_DEBUG(getLogger(), "Inserting entity of type {} named {} with id {}", type_info.name, name, toString(id));
|
||||
@ -108,11 +121,11 @@ std::optional<UUID> ReplicatedAccessStorage::insertImpl(const AccessEntityPtr &
|
||||
retryOnZooKeeperUserError(10, [&]{ ok = insertZooKeeper(zookeeper, id, new_entity, replace_if_exists, throw_if_exists); });
|
||||
|
||||
if (!ok)
|
||||
return std::nullopt;
|
||||
return false;
|
||||
|
||||
std::lock_guard lock{mutex};
|
||||
refreshEntityNoLock(zookeeper, id);
|
||||
return id;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@ -600,4 +613,19 @@ AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id, bool throw_if
|
||||
return entry.entity;
|
||||
}
|
||||
|
||||
void ReplicatedAccessStorage::insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr<IRestoreCoordination> restore_coordination)
|
||||
{
|
||||
if (!isRestoreAllowed())
|
||||
throwRestoreNotAllowed();
|
||||
|
||||
if (!restore_coordination->acquireReplicatedAccessStorage(zookeeper_path))
|
||||
return;
|
||||
|
||||
bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace);
|
||||
bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate);
|
||||
|
||||
for (const auto & [id, entity] : entities_from_backup)
|
||||
insertWithID(id, entity, replace_if_exists, throw_if_exists);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <mutex>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <base/defines.h>
|
||||
#include <base/scope_guard.h>
|
||||
|
||||
#include <Common/ThreadPool.h>
|
||||
@ -26,7 +27,7 @@ class ReplicatedAccessStorage : public IAccessStorage
|
||||
public:
|
||||
static constexpr char STORAGE_TYPE[] = "replicated";
|
||||
|
||||
ReplicatedAccessStorage(const String & storage_name, const String & zookeeper_path, zkutil::GetZooKeeper get_zookeeper, AccessChangesNotifier & changes_notifier_);
|
||||
ReplicatedAccessStorage(const String & storage_name, const String & zookeeper_path, zkutil::GetZooKeeper get_zookeeper, AccessChangesNotifier & changes_notifier_, bool allow_backup);
|
||||
virtual ~ReplicatedAccessStorage() override;
|
||||
|
||||
const char * getStorageType() const override { return STORAGE_TYPE; }
|
||||
@ -36,6 +37,9 @@ public:
|
||||
|
||||
bool exists(const UUID & id) const override;
|
||||
|
||||
bool isBackupAllowed() const override { return backup_allowed; }
|
||||
void insertFromBackup(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr<IRestoreCoordination> restore_coordination) override;
|
||||
|
||||
private:
|
||||
String zookeeper_path;
|
||||
zkutil::GetZooKeeper get_zookeeper;
|
||||
@ -50,6 +54,7 @@ private:
|
||||
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
||||
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
||||
|
||||
bool insertWithID(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists);
|
||||
bool insertZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
|
||||
bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists);
|
||||
bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
||||
@ -66,10 +71,10 @@ private:
|
||||
bool refresh();
|
||||
void refreshEntities(const zkutil::ZooKeeperPtr & zookeeper);
|
||||
void refreshEntity(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id);
|
||||
void refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id);
|
||||
void refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) TSA_REQUIRES(mutex);
|
||||
|
||||
void setEntityNoLock(const UUID & id, const AccessEntityPtr & entity);
|
||||
void removeEntityNoLock(const UUID & id);
|
||||
void setEntityNoLock(const UUID & id, const AccessEntityPtr & entity) TSA_REQUIRES(mutex);
|
||||
void removeEntityNoLock(const UUID & id) TSA_REQUIRES(mutex);
|
||||
|
||||
struct Entry
|
||||
{
|
||||
@ -82,8 +87,9 @@ private:
|
||||
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
|
||||
mutable std::mutex mutex;
|
||||
std::unordered_map<UUID, Entry> entries_by_id;
|
||||
std::unordered_map<String, Entry *> entries_by_name_and_type[static_cast<size_t>(AccessEntityType::MAX)];
|
||||
std::unordered_map<UUID, Entry> entries_by_id TSA_GUARDED_BY(mutex);
|
||||
std::unordered_map<String, Entry *> entries_by_name_and_type[static_cast<size_t>(AccessEntityType::MAX)] TSA_GUARDED_BY(mutex);
|
||||
AccessChangesNotifier & changes_notifier;
|
||||
bool backup_allowed = false;
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#include <Access/Role.h>
|
||||
#include <base/insertAtEnd.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -12,4 +13,18 @@ bool Role::equal(const IAccessEntity & other) const
|
||||
return (access == other_role.access) && (granted_roles == other_role.granted_roles) && (settings == other_role.settings);
|
||||
}
|
||||
|
||||
std::vector<UUID> Role::findDependencies() const
|
||||
{
|
||||
std::vector<UUID> res;
|
||||
insertAtEnd(res, granted_roles.findDependencies());
|
||||
insertAtEnd(res, settings.findDependencies());
|
||||
return res;
|
||||
}
|
||||
|
||||
void Role::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||
{
|
||||
granted_roles.replaceDependencies(old_to_new_ids);
|
||||
settings.replaceDependencies(old_to_new_ids);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -19,6 +19,10 @@ struct Role : public IAccessEntity
|
||||
std::shared_ptr<IAccessEntity> clone() const override { return cloneImpl<Role>(); }
|
||||
static constexpr const auto TYPE = AccessEntityType::ROLE;
|
||||
AccessEntityType getType() const override { return TYPE; }
|
||||
|
||||
std::vector<UUID> findDependencies() const override;
|
||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
||||
bool isBackupAllowed() const override { return settings.isBackupAllowed(); }
|
||||
};
|
||||
|
||||
using RolePtr = std::shared_ptr<const Role>;
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <boost/range/algorithm/set_algorithm.hpp>
|
||||
#include <boost/range/algorithm/copy.hpp>
|
||||
#include <boost/range/algorithm_ext/push_back.hpp>
|
||||
#include <base/sort.h>
|
||||
|
||||
@ -286,4 +287,54 @@ bool operator ==(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs)
|
||||
return (lhs.all == rhs.all) && (lhs.ids == rhs.ids) && (lhs.except_ids == rhs.except_ids);
|
||||
}
|
||||
|
||||
std::vector<UUID> RolesOrUsersSet::findDependencies() const
|
||||
{
|
||||
std::vector<UUID> res;
|
||||
boost::range::copy(ids, std::back_inserter(res));
|
||||
boost::range::copy(except_ids, std::back_inserter(res));
|
||||
return res;
|
||||
}
|
||||
|
||||
void RolesOrUsersSet::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||
{
|
||||
std::vector<UUID> new_ids;
|
||||
|
||||
for (auto it = ids.begin(); it != ids.end();)
|
||||
{
|
||||
auto id = *it;
|
||||
auto it_new_id = old_to_new_ids.find(id);
|
||||
if (it_new_id != old_to_new_ids.end())
|
||||
{
|
||||
auto new_id = it_new_id->second;
|
||||
new_ids.push_back(new_id);
|
||||
it = ids.erase(it);
|
||||
}
|
||||
else
|
||||
{
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
boost::range::copy(new_ids, std::inserter(ids, ids.end()));
|
||||
new_ids.clear();
|
||||
|
||||
for (auto it = except_ids.begin(); it != except_ids.end();)
|
||||
{
|
||||
auto id = *it;
|
||||
auto it_new_id = old_to_new_ids.find(id);
|
||||
if (it_new_id != old_to_new_ids.end())
|
||||
{
|
||||
auto new_id = it_new_id->second;
|
||||
new_ids.push_back(new_id);
|
||||
it = except_ids.erase(it);
|
||||
}
|
||||
else
|
||||
{
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
boost::range::copy(new_ids, std::inserter(except_ids, except_ids.end()));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -62,6 +63,9 @@ struct RolesOrUsersSet
|
||||
friend bool operator ==(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs);
|
||||
friend bool operator !=(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs) { return !(lhs == rhs); }
|
||||
|
||||
std::vector<UUID> findDependencies() const;
|
||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
|
||||
|
||||
bool all = false;
|
||||
boost::container::flat_set<UUID> ids;
|
||||
boost::container::flat_set<UUID> except_ids;
|
||||
|
||||
@ -58,4 +58,14 @@ bool RowPolicy::equal(const IAccessEntity & other) const
|
||||
&& restrictive == other_policy.restrictive && (to_roles == other_policy.to_roles);
|
||||
}
|
||||
|
||||
std::vector<UUID> RowPolicy::findDependencies() const
|
||||
{
|
||||
return to_roles.findDependencies();
|
||||
}
|
||||
|
||||
void RowPolicy::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||
{
|
||||
to_roles.replaceDependencies(old_to_new_ids);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -46,6 +46,10 @@ struct RowPolicy : public IAccessEntity
|
||||
static constexpr const auto TYPE = AccessEntityType::ROW_POLICY;
|
||||
AccessEntityType getType() const override { return TYPE; }
|
||||
|
||||
std::vector<UUID> findDependencies() const override;
|
||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
||||
bool isBackupAllowed() const override { return true; }
|
||||
|
||||
/// Which roles or users should use this row policy.
|
||||
RolesOrUsersSet to_roles;
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#include <Access/SettingsProfile.h>
|
||||
#include <base/insertAtEnd.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -12,4 +13,18 @@ bool SettingsProfile::equal(const IAccessEntity & other) const
|
||||
return (elements == other_profile.elements) && (to_roles == other_profile.to_roles);
|
||||
}
|
||||
|
||||
std::vector<UUID> SettingsProfile::findDependencies() const
|
||||
{
|
||||
std::vector<UUID> res;
|
||||
insertAtEnd(res, elements.findDependencies());
|
||||
insertAtEnd(res, to_roles.findDependencies());
|
||||
return res;
|
||||
}
|
||||
|
||||
void SettingsProfile::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||
{
|
||||
elements.replaceDependencies(old_to_new_ids);
|
||||
to_roles.replaceDependencies(old_to_new_ids);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -20,6 +20,10 @@ struct SettingsProfile : public IAccessEntity
|
||||
std::shared_ptr<IAccessEntity> clone() const override { return cloneImpl<SettingsProfile>(); }
|
||||
static constexpr const auto TYPE = AccessEntityType::SETTINGS_PROFILE;
|
||||
AccessEntityType getType() const override { return TYPE; }
|
||||
|
||||
std::vector<UUID> findDependencies() const override;
|
||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
||||
bool isBackupAllowed() const override { return elements.isBackupAllowed(); }
|
||||
};
|
||||
|
||||
using SettingsProfilePtr = std::shared_ptr<const SettingsProfile>;
|
||||
|
||||
@ -12,6 +12,13 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
constexpr const char ALLOW_BACKUP_SETTING_NAME[] = "allow_backup";
|
||||
}
|
||||
|
||||
|
||||
SettingsProfileElement::SettingsProfileElement(const ASTSettingsProfileElement & ast)
|
||||
{
|
||||
init(ast, nullptr);
|
||||
@ -41,7 +48,10 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A
|
||||
|
||||
/// Optionally check if a setting with that name is allowed.
|
||||
if (access_control)
|
||||
access_control->checkSettingNameIsAllowed(setting_name);
|
||||
{
|
||||
if (setting_name != ALLOW_BACKUP_SETTING_NAME)
|
||||
access_control->checkSettingNameIsAllowed(setting_name);
|
||||
}
|
||||
|
||||
value = ast.value;
|
||||
min_value = ast.min_value;
|
||||
@ -127,6 +137,36 @@ std::shared_ptr<ASTSettingsProfileElements> SettingsProfileElements::toASTWithNa
|
||||
}
|
||||
|
||||
|
||||
std::vector<UUID> SettingsProfileElements::findDependencies() const
|
||||
{
|
||||
std::vector<UUID> res;
|
||||
for (const auto & element : *this)
|
||||
{
|
||||
if (element.parent_profile)
|
||||
res.push_back(*element.parent_profile);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
void SettingsProfileElements::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||
{
|
||||
for (auto & element : *this)
|
||||
{
|
||||
if (element.parent_profile)
|
||||
{
|
||||
auto id = *element.parent_profile;
|
||||
auto it_new_id = old_to_new_ids.find(id);
|
||||
if (it_new_id != old_to_new_ids.end())
|
||||
{
|
||||
auto new_id = it_new_id->second;
|
||||
element.parent_profile = new_id;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void SettingsProfileElements::merge(const SettingsProfileElements & other)
|
||||
{
|
||||
insert(end(), other.begin(), other.end());
|
||||
@ -138,8 +178,11 @@ Settings SettingsProfileElements::toSettings() const
|
||||
Settings res;
|
||||
for (const auto & elem : *this)
|
||||
{
|
||||
if (!elem.setting_name.empty() && !elem.value.isNull())
|
||||
res.set(elem.setting_name, elem.value);
|
||||
if (!elem.setting_name.empty() && (elem.setting_name != ALLOW_BACKUP_SETTING_NAME))
|
||||
{
|
||||
if (!elem.value.isNull())
|
||||
res.set(elem.setting_name, elem.value);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
@ -149,8 +192,11 @@ SettingsChanges SettingsProfileElements::toSettingsChanges() const
|
||||
SettingsChanges res;
|
||||
for (const auto & elem : *this)
|
||||
{
|
||||
if (!elem.setting_name.empty() && !elem.value.isNull())
|
||||
res.push_back({elem.setting_name, elem.value});
|
||||
if (!elem.setting_name.empty() && (elem.setting_name != ALLOW_BACKUP_SETTING_NAME))
|
||||
{
|
||||
if (!elem.value.isNull())
|
||||
res.push_back({elem.setting_name, elem.value});
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
@ -160,7 +206,7 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints(const AccessC
|
||||
SettingsConstraints res{access_control};
|
||||
for (const auto & elem : *this)
|
||||
{
|
||||
if (!elem.setting_name.empty())
|
||||
if (!elem.setting_name.empty() && (elem.setting_name != ALLOW_BACKUP_SETTING_NAME))
|
||||
{
|
||||
if (!elem.min_value.isNull())
|
||||
res.setMinValue(elem.setting_name, elem.min_value);
|
||||
@ -189,5 +235,14 @@ std::vector<UUID> SettingsProfileElements::toProfileIDs() const
|
||||
return res;
|
||||
}
|
||||
|
||||
bool SettingsProfileElements::isBackupAllowed() const
|
||||
{
|
||||
for (const auto & setting : *this)
|
||||
{
|
||||
if (setting.setting_name == ALLOW_BACKUP_SETTING_NAME)
|
||||
return static_cast<bool>(SettingFieldBool{setting.value});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Core/Field.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
|
||||
@ -57,12 +58,17 @@ public:
|
||||
std::shared_ptr<ASTSettingsProfileElements> toAST() const;
|
||||
std::shared_ptr<ASTSettingsProfileElements> toASTWithNames(const AccessControl & access_control) const;
|
||||
|
||||
std::vector<UUID> findDependencies() const;
|
||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids);
|
||||
|
||||
void merge(const SettingsProfileElements & other);
|
||||
|
||||
Settings toSettings() const;
|
||||
SettingsChanges toSettingsChanges() const;
|
||||
SettingsConstraints toSettingsConstraints(const AccessControl & access_control) const;
|
||||
std::vector<UUID> toProfileIDs() const;
|
||||
|
||||
bool isBackupAllowed() const;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <Access/User.h>
|
||||
#include <Core/Protocol.h>
|
||||
#include <base/insertAtEnd.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -31,4 +32,22 @@ void User::setName(const String & name_)
|
||||
name = name_;
|
||||
}
|
||||
|
||||
std::vector<UUID> User::findDependencies() const
|
||||
{
|
||||
std::vector<UUID> res;
|
||||
insertAtEnd(res, default_roles.findDependencies());
|
||||
insertAtEnd(res, granted_roles.findDependencies());
|
||||
insertAtEnd(res, grantees.findDependencies());
|
||||
insertAtEnd(res, settings.findDependencies());
|
||||
return res;
|
||||
}
|
||||
|
||||
void User::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||
{
|
||||
default_roles.replaceDependencies(old_to_new_ids);
|
||||
granted_roles.replaceDependencies(old_to_new_ids);
|
||||
grantees.replaceDependencies(old_to_new_ids);
|
||||
settings.replaceDependencies(old_to_new_ids);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -29,6 +29,10 @@ struct User : public IAccessEntity
|
||||
static constexpr const auto TYPE = AccessEntityType::USER;
|
||||
AccessEntityType getType() const override { return TYPE; }
|
||||
void setName(const String & name_) override;
|
||||
|
||||
std::vector<UUID> findDependencies() const override;
|
||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
||||
bool isBackupAllowed() const override { return settings.isBackupAllowed(); }
|
||||
};
|
||||
|
||||
using UserPtr = std::shared_ptr<const User>;
|
||||
|
||||
@ -523,8 +523,11 @@ namespace
|
||||
}
|
||||
}
|
||||
|
||||
UsersConfigAccessStorage::UsersConfigAccessStorage(const String & storage_name_, AccessControl & access_control_)
|
||||
: IAccessStorage(storage_name_), access_control(access_control_), memory_storage(storage_name_, access_control.getChangesNotifier())
|
||||
UsersConfigAccessStorage::UsersConfigAccessStorage(const String & storage_name_, AccessControl & access_control_, bool allow_backup_)
|
||||
: IAccessStorage(storage_name_)
|
||||
, access_control(access_control_)
|
||||
, memory_storage(storage_name_, access_control.getChangesNotifier(), false)
|
||||
, backup_allowed(allow_backup_)
|
||||
{
|
||||
}
|
||||
|
||||
@ -655,9 +658,9 @@ AccessEntityPtr UsersConfigAccessStorage::readImpl(const UUID & id, bool throw_i
|
||||
}
|
||||
|
||||
|
||||
std::optional<String> UsersConfigAccessStorage::readNameImpl(const UUID & id, bool throw_if_not_exists) const
|
||||
std::optional<std::pair<String, AccessEntityType>> UsersConfigAccessStorage::readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const
|
||||
{
|
||||
return memory_storage.readName(id, throw_if_not_exists);
|
||||
return memory_storage.readNameWithType(id, throw_if_not_exists);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -22,7 +22,7 @@ public:
|
||||
|
||||
static constexpr char STORAGE_TYPE[] = "users.xml";
|
||||
|
||||
UsersConfigAccessStorage(const String & storage_name_, AccessControl & access_control_);
|
||||
UsersConfigAccessStorage(const String & storage_name_, AccessControl & access_control_, bool allow_backup_);
|
||||
~UsersConfigAccessStorage() override;
|
||||
|
||||
const char * getStorageType() const override { return STORAGE_TYPE; }
|
||||
@ -44,17 +44,20 @@ public:
|
||||
|
||||
bool exists(const UUID & id) const override;
|
||||
|
||||
bool isBackupAllowed() const override { return backup_allowed; }
|
||||
|
||||
private:
|
||||
void parseFromConfig(const Poco::Util::AbstractConfiguration & config);
|
||||
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
||||
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
std::optional<String> readNameImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
|
||||
AccessControl & access_control;
|
||||
MemoryAccessStorage memory_storage;
|
||||
String path;
|
||||
std::unique_ptr<ConfigReloader> config_reloader;
|
||||
bool backup_allowed = false;
|
||||
mutable std::mutex load_mutex;
|
||||
};
|
||||
}
|
||||
|
||||
@ -49,7 +49,7 @@ TEST(AccessRights, Union)
|
||||
"GRANT INSERT ON *.*, "
|
||||
"GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, "
|
||||
"CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, "
|
||||
"TRUNCATE, OPTIMIZE, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, "
|
||||
"TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, "
|
||||
"SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
|
||||
"SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
|
||||
"SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user